2023-11-12 09:26:03,922 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3eb25e1a 2023-11-12 09:26:03,936 main DEBUG Took 0.011587 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-11-12 09:26:03,936 main DEBUG PluginManager 'Core' found 129 plugins 2023-11-12 09:26:03,936 main DEBUG PluginManager 'Level' found 0 plugins 2023-11-12 09:26:03,937 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-11-12 09:26:03,939 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:26:03,950 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-11-12 09:26:03,968 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:26:03,971 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:26:03,972 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,972 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:26:03,973 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:26:03,974 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:26:03,975 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:26:03,976 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:26:03,976 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:26:03,977 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:26:03,978 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:26:03,978 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:26:03,979 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:26:03,980 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,980 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-11-12 09:26:03,982 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:26:03,984 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-11-12 09:26:03,986 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-11-12 09:26:03,986 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-11-12 09:26:03,987 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-11-12 09:26:03,987 main DEBUG PluginManager 'Converter' found 47 plugins 2023-11-12 09:26:04,001 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-11-12 09:26:04,004 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-11-12 09:26:04,005 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-11-12 09:26:04,005 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-11-12 09:26:04,006 main DEBUG createAppenders(={Console}) 2023-11-12 09:26:04,006 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3eb25e1a initialized 2023-11-12 09:26:04,007 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3eb25e1a 2023-11-12 09:26:04,007 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3eb25e1a OK. 2023-11-12 09:26:04,007 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-11-12 09:26:04,008 main DEBUG OutputStream closed 2023-11-12 09:26:04,008 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-11-12 09:26:04,008 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-11-12 09:26:04,008 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@59fd97a8 OK 2023-11-12 09:26:04,059 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-11-12 09:26:04,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-11-12 09:26:04,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-11-12 09:26:04,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-11-12 09:26:04,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-11-12 09:26:04,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-11-12 09:26:04,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-11-12 09:26:04,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-11-12 09:26:04,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-11-12 09:26:04,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-11-12 09:26:04,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-11-12 09:26:04,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-11-12 09:26:04,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-11-12 09:26:04,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-11-12 09:26:04,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-11-12 09:26:04,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-11-12 09:26:04,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-11-12 09:26:04,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-11-12 09:26:04,074 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-12 09:26:04,074 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5403f35f) with optional ClassLoader: null 2023-11-12 09:26:04,075 main DEBUG Shutdown hook enabled. Registering a new one. 2023-11-12 09:26:04,076 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@5403f35f] started OK. 2023-11-12T09:26:04,112 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestFullRestore timeout: 13 mins 2023-11-12 09:26:04,116 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-11-12 09:26:04,116 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-12T09:26:04,456 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb 2023-11-12T09:26:04,461 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-12T09:26:04,464 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:26:04,465 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-11-12T09:26:04,469 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/cluster_2df797c1-14cb-183c-6424-e9a9271897e8, deleteOnExit=true 2023-11-12T09:26:04,469 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-11-12T09:26:04,469 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/test.cache.data in system properties and HBase conf 2023-11-12T09:26:04,470 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop.tmp.dir in system properties and HBase conf 2023-11-12T09:26:04,470 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop.log.dir in system properties and HBase conf 2023-11-12T09:26:04,471 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-12T09:26:04,471 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-12T09:26:04,471 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-12T09:26:04,592 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-11-12T09:26:05,175 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-11-12T09:26:05,180 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:26:05,180 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:26:05,181 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-12T09:26:05,181 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:26:05,181 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-12T09:26:05,182 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-12T09:26:05,182 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:26:05,182 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:26:05,182 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-12T09:26:05,183 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/nfs.dump.dir in system properties and HBase conf 2023-11-12T09:26:05,183 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/java.io.tmpdir in system properties and HBase conf 2023-11-12T09:26:05,183 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:26:05,184 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-12T09:26:05,184 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-11-12T09:26:05,684 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:26:05,688 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:26:06,002 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-11-12T09:26:06,155 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@3abc60ad via org.mortbay.log.Slf4jLog 2023-11-12T09:26:06,176 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:26:06,233 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:26:06,272 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/java.io.tmpdir/Jetty_localhost_44661_hdfs____.wjnxid/webapp 2023-11-12T09:26:06,407 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:44661 2023-11-12T09:26:06,432 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:26:06,432 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:26:07,063 WARN [Listener at localhost/34955 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:26:07,151 WARN [Listener at localhost/34955 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-12T09:26:07,177 WARN [Listener at localhost/34955 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:26:07,184 INFO [Listener at localhost/34955 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:26:07,190 INFO [Listener at localhost/34955 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/java.io.tmpdir/Jetty_localhost_35057_datanode____etygbb/webapp 2023-11-12T09:26:07,293 INFO [Listener at localhost/34955 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35057 2023-11-12T09:26:07,575 WARN [Listener at localhost/39325 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:26:07,879 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-12T09:26:08,039 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xa0607210e01fdfdf: Processing first storage report for DS-32cb1215-5008-40d4-a569-9a97b9569b71 from datanode d9010a15-a2e1-4fc3-a1cc-a04c2149c14d 2023-11-12T09:26:08,040 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xa0607210e01fdfdf: from storage DS-32cb1215-5008-40d4-a569-9a97b9569b71 node DatanodeRegistration(127.0.0.1:36711, datanodeUuid=d9010a15-a2e1-4fc3-a1cc-a04c2149c14d, infoPort=36615, infoSecurePort=0, ipcPort=39325, storageInfo=lv=-57;cid=testClusterID;nsid=595646075;c=1699781165767), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-11-12T09:26:08,040 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xa0607210e01fdfdf: Processing first storage report for DS-aae2b561-6af6-4585-b865-ef319c11487f from datanode d9010a15-a2e1-4fc3-a1cc-a04c2149c14d 2023-11-12T09:26:08,041 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xa0607210e01fdfdf: from storage DS-aae2b561-6af6-4585-b865-ef319c11487f node DatanodeRegistration(127.0.0.1:36711, datanodeUuid=d9010a15-a2e1-4fc3-a1cc-a04c2149c14d, infoPort=36615, infoSecurePort=0, ipcPort=39325, storageInfo=lv=-57;cid=testClusterID;nsid=595646075;c=1699781165767), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:26:08,104 DEBUG [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb 2023-11-12T09:26:08,257 INFO [Listener at localhost/39325 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/cluster_2df797c1-14cb-183c-6424-e9a9271897e8/zookeeper_0, clientPort=56042, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/cluster_2df797c1-14cb-183c-6424-e9a9271897e8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/cluster_2df797c1-14cb-183c-6424-e9a9271897e8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2023-11-12T09:26:08,278 INFO [Listener at localhost/39325 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=56042 2023-11-12T09:26:08,285 INFO [Listener at localhost/39325 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:26:08,288 INFO [Listener at localhost/39325 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:26:08,980 INFO [Listener at localhost/39325 {}] util.FSUtils(462): Created version file at hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa with version=8 2023-11-12T09:26:08,980 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging 2023-11-12T09:26:08,998 DEBUG [Listener at localhost/39325 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-11-12T09:26:08,998 DEBUG [Listener at localhost/39325 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-11-12T09:26:08,998 DEBUG [Listener at localhost/39325 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-11-12T09:26:08,998 DEBUG [Listener at localhost/39325 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-11-12T09:26:09,228 DEBUG [Listener at localhost/39325 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-11-12T09:26:10,020 INFO [Listener at localhost/39325 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-11-12T09:26:10,068 INFO [Listener at localhost/39325 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:26:10,068 INFO [Listener at localhost/39325 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-12T09:26:10,069 INFO [Listener at localhost/39325 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-12T09:26:10,069 INFO [Listener at localhost/39325 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:26:10,069 INFO [Listener at localhost/39325 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-12T09:26:10,222 INFO [Listener at localhost/39325 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-12T09:26:10,353 INFO [Listener at localhost/39325 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-11-12T09:26:10,371 DEBUG [Listener at localhost/39325 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-11-12T09:26:10,377 INFO [Listener at localhost/39325 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-12T09:26:10,425 DEBUG [Listener at localhost/39325 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 12993 (auto-detected) 2023-11-12T09:26:10,427 DEBUG [Listener at localhost/39325 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:0f:ff:fe:94:aa:bf (auto-detected) 2023-11-12T09:26:10,468 INFO [Listener at localhost/39325 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:43639 2023-11-12T09:26:10,482 INFO [Listener at localhost/39325 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:26:10,484 INFO [Listener at localhost/39325 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:26:10,506 INFO [Listener at localhost/39325 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:43639 connecting to ZooKeeper ensemble=127.0.0.1:56042 2023-11-12T09:26:10,558 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:436390x0, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-12T09:26:10,562 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:43639-0x1004cf43a5e0000 connected 2023-11-12T09:26:10,607 DEBUG [Listener at localhost/39325 {}] zookeeper.ZKUtil(113): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-12T09:26:10,611 DEBUG [Listener at localhost/39325 {}] zookeeper.ZKUtil(113): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:26:10,615 DEBUG [Listener at localhost/39325 {}] zookeeper.ZKUtil(113): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-12T09:26:10,628 DEBUG [Listener at localhost/39325 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43639 2023-11-12T09:26:10,628 DEBUG [Listener at localhost/39325 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43639 2023-11-12T09:26:10,629 DEBUG [Listener at localhost/39325 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43639 2023-11-12T09:26:10,630 DEBUG [Listener at localhost/39325 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43639 2023-11-12T09:26:10,630 DEBUG [Listener at localhost/39325 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43639 2023-11-12T09:26:10,696 INFO [Listener at localhost/39325 {}] log.Log(170): Logging initialized @7549ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-11-12T09:26:10,817 INFO [Listener at localhost/39325 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-12T09:26:10,818 INFO [Listener at localhost/39325 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-12T09:26:10,818 INFO [Listener at localhost/39325 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-12T09:26:10,821 INFO [Listener at localhost/39325 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-11-12T09:26:10,821 INFO [Listener at localhost/39325 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-12T09:26:10,821 INFO [Listener at localhost/39325 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-12T09:26:10,824 INFO [Listener at localhost/39325 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-12T09:26:10,954 INFO [Listener at localhost/39325 {}] http.HttpServer(1227): Jetty bound to port 34843 2023-11-12T09:26:10,956 INFO [Listener at localhost/39325 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:26:10,990 INFO [Listener at localhost/39325 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:26:10,994 INFO [Listener at localhost/39325 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@56e2e346{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop.log.dir/,AVAILABLE} 2023-11-12T09:26:10,994 INFO [Listener at localhost/39325 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:26:10,995 INFO [Listener at localhost/39325 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3645ed1f{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-12T09:26:11,057 INFO [Listener at localhost/39325 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-12T09:26:11,070 INFO [Listener at localhost/39325 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-12T09:26:11,071 INFO [Listener at localhost/39325 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-12T09:26:11,073 INFO [Listener at localhost/39325 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-11-12T09:26:11,080 INFO [Listener at localhost/39325 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:26:12,277 INFO [Listener at localhost/39325 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@65329596{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-12T09:26:12,288 INFO [Listener at localhost/39325 {}] server.AbstractConnector(333): Started ServerConnector@45e81754{HTTP/1.1, (http/1.1)}{0.0.0.0:34843} 2023-11-12T09:26:12,289 INFO [Listener at localhost/39325 {}] server.Server(415): Started @9142ms 2023-11-12T09:26:12,292 INFO [Listener at localhost/39325 {}] master.HMaster(486): hbase.rootdir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, hbase.cluster.distributed=false 2023-11-12T09:26:12,353 INFO [Listener at localhost/39325 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-11-12T09:26:12,354 INFO [Listener at localhost/39325 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:26:12,354 INFO [Listener at localhost/39325 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-12T09:26:12,354 INFO [Listener at localhost/39325 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-12T09:26:12,354 INFO [Listener at localhost/39325 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:26:12,354 INFO [Listener at localhost/39325 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-12T09:26:12,362 INFO [Listener at localhost/39325 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-12T09:26:12,364 INFO [Listener at localhost/39325 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-12T09:26:12,365 INFO [Listener at localhost/39325 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:36287 2023-11-12T09:26:12,368 INFO [Listener at localhost/39325 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-11-12T09:26:12,375 DEBUG [Listener at localhost/39325 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-11-12T09:26:12,376 INFO [Listener at localhost/39325 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:26:12,378 INFO [Listener at localhost/39325 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:26:12,380 INFO [Listener at localhost/39325 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:36287 connecting to ZooKeeper ensemble=127.0.0.1:56042 2023-11-12T09:26:12,384 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:362870x0, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-12T09:26:12,385 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:36287-0x1004cf43a5e0001 connected 2023-11-12T09:26:12,385 DEBUG [Listener at localhost/39325 {}] zookeeper.ZKUtil(113): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-12T09:26:12,387 DEBUG [Listener at localhost/39325 {}] zookeeper.ZKUtil(113): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:26:12,388 DEBUG [Listener at localhost/39325 {}] zookeeper.ZKUtil(113): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-12T09:26:12,388 DEBUG [Listener at localhost/39325 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36287 2023-11-12T09:26:12,388 DEBUG [Listener at localhost/39325 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36287 2023-11-12T09:26:12,389 DEBUG [Listener at localhost/39325 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36287 2023-11-12T09:26:12,389 DEBUG [Listener at localhost/39325 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36287 2023-11-12T09:26:12,390 DEBUG [Listener at localhost/39325 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36287 2023-11-12T09:26:12,393 INFO [Listener at localhost/39325 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-12T09:26:12,393 INFO [Listener at localhost/39325 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-12T09:26:12,393 INFO [Listener at localhost/39325 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-12T09:26:12,394 INFO [Listener at localhost/39325 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-11-12T09:26:12,394 INFO [Listener at localhost/39325 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-12T09:26:12,394 INFO [Listener at localhost/39325 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-12T09:26:12,394 INFO [Listener at localhost/39325 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-12T09:26:12,397 INFO [Listener at localhost/39325 {}] http.HttpServer(1227): Jetty bound to port 42423 2023-11-12T09:26:12,397 INFO [Listener at localhost/39325 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:26:12,398 INFO [Listener at localhost/39325 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:26:12,399 INFO [Listener at localhost/39325 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@67098989{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop.log.dir/,AVAILABLE} 2023-11-12T09:26:12,399 INFO [Listener at localhost/39325 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:26:12,399 INFO [Listener at localhost/39325 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@571c5e22{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-12T09:26:12,411 INFO [Listener at localhost/39325 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-12T09:26:12,412 INFO [Listener at localhost/39325 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-12T09:26:12,412 INFO [Listener at localhost/39325 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-12T09:26:12,412 INFO [Listener at localhost/39325 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-11-12T09:26:12,413 INFO [Listener at localhost/39325 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:26:12,418 INFO [Listener at localhost/39325 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@405ea105{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-12T09:26:12,419 INFO [Listener at localhost/39325 {}] server.AbstractConnector(333): Started ServerConnector@342cc1cb{HTTP/1.1, (http/1.1)}{0.0.0.0:42423} 2023-11-12T09:26:12,419 INFO [Listener at localhost/39325 {}] server.Server(415): Started @9272ms 2023-11-12T09:26:12,424 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:26:12,429 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@28909016{HTTP/1.1, (http/1.1)}{0.0.0.0:46101} 2023-11-12T09:26:12,429 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @9282ms 2023-11-12T09:26:12,429 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2389): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,43639,1699781169211 2023-11-12T09:26:12,442 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:26:12,442 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:26:12,446 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,43639,1699781169211 2023-11-12T09:26:12,451 DEBUG [M:0;jenkins-hbase3:43639 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:43639 2023-11-12T09:26:12,467 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-12T09:26:12,467 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-12T09:26:12,467 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:12,467 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:12,468 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-12T09:26:12,470 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,43639,1699781169211 from backup master directory 2023-11-12T09:26:12,470 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-12T09:26:12,474 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,43639,1699781169211 2023-11-12T09:26:12,474 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:26:12,474 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:26:12,475 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-12T09:26:12,475 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,43639,1699781169211 2023-11-12T09:26:12,479 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-11-12T09:26:12,480 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-11-12T09:26:12,966 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase.id with ID: 2a167737-49b4-4d1b-a684-e0cae75fde82 2023-11-12T09:26:13,014 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:26:13,031 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:13,031 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:13,477 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:26:13,481 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-11-12T09:26:13,502 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-11-12T09:26:13,502 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-11-12T09:26:13,504 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:26:13,516 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:26:13,518 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:26:13,598 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store 2023-11-12T09:26:14,024 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-11-12T09:26:14,025 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:14,025 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-12T09:26:14,025 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:26:14,025 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:26:14,025 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-12T09:26:14,025 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:26:14,025 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:26:14,026 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:26:14,028 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/.initializing 2023-11-12T09:26:14,028 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/WALs/jenkins-hbase3.apache.org,43639,1699781169211 2023-11-12T09:26:14,038 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-12T09:26:14,052 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C43639%2C1699781169211, suffix=, logDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/WALs/jenkins-hbase3.apache.org,43639,1699781169211, archiveDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/oldWALs, maxLogs=10 2023-11-12T09:26:14,083 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/WALs/jenkins-hbase3.apache.org,43639,1699781169211/jenkins-hbase3.apache.org%2C43639%2C1699781169211.1699781174060, exclude list is [], retry=0 2023-11-12T09:26:14,113 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:26:14,121 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:499) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:26:14,199 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/WALs/jenkins-hbase3.apache.org,43639,1699781169211/jenkins-hbase3.apache.org%2C43639%2C1699781169211.1699781174060 2023-11-12T09:26:14,199 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:26:14,200 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:26:14,201 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:14,205 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:26:14,206 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:26:14,284 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:26:14,325 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-11-12T09:26:14,332 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:14,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:26:14,336 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:26:14,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-11-12T09:26:14,339 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:14,340 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:14,340 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:26:14,343 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-11-12T09:26:14,343 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:14,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:14,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:26:14,347 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-11-12T09:26:14,347 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:14,348 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:14,352 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:26:14,353 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:26:14,369 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-11-12T09:26:14,373 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:26:14,377 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:26:14,378 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11125576640, jitterRate=0.036150068044662476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-11-12T09:26:14,383 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:26:14,384 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-11-12T09:26:14,419 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@88e1575, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:14,475 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(861): No meta location available on zookeeper, skip migrating... 2023-11-12T09:26:14,495 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-11-12T09:26:14,496 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-11-12T09:26:14,499 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-11-12T09:26:14,500 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-11-12T09:26:14,506 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 6 msec 2023-11-12T09:26:14,507 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-11-12T09:26:14,541 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-11-12T09:26:14,558 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-11-12T09:26:14,560 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-11-12T09:26:14,564 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-11-12T09:26:14,565 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-11-12T09:26:14,567 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-11-12T09:26:14,570 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-11-12T09:26:14,577 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-11-12T09:26:14,578 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-11-12T09:26:14,579 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-11-12T09:26:14,581 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-11-12T09:26:14,597 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-11-12T09:26:14,598 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-11-12T09:26:14,606 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-12T09:26:14,606 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-12T09:26:14,606 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:14,606 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:14,607 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(805): Active/primary master=jenkins-hbase3.apache.org,43639,1699781169211, sessionid=0x1004cf43a5e0000, setting cluster-up flag (Was=false) 2023-11-12T09:26:14,618 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-11-12T09:26:14,630 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:14,630 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:14,637 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-11-12T09:26:14,639 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43639,1699781169211 2023-11-12T09:26:14,642 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:14,642 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:14,648 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-11-12T09:26:14,650 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43639,1699781169211 2023-11-12T09:26:14,656 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:14,656 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:14,661 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-11-12T09:26:14,663 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43639,1699781169211 2023-11-12T09:26:14,735 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:36287 2023-11-12T09:26:14,737 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(997): ClusterId : 2a167737-49b4-4d1b-a684-e0cae75fde82 2023-11-12T09:26:14,739 INFO [RS:0;jenkins-hbase3:36287 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-11-12T09:26:14,742 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-11-12T09:26:14,749 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-11-12T09:26:14,749 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-11-12T09:26:14,752 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-11-12T09:26:14,752 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-11-12T09:26:14,755 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-11-12T09:26:14,758 DEBUG [RS:0;jenkins-hbase3:36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57e420a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:14,759 DEBUG [RS:0;jenkins-hbase3:36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a528e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-11-12T09:26:14,765 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-11-12T09:26:14,765 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-11-12T09:26:14,765 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-11-12T09:26:14,769 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,43639,1699781169211 with isa=jenkins-hbase3.apache.org/172.31.12.81:36287, startcode=1699781172353 2023-11-12T09:26:14,792 DEBUG [RS:0;jenkins-hbase3:36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-12T09:26:14,825 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-11-12T09:26:14,841 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-11-12T09:26:14,850 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-11-12T09:26:14,861 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,43639,1699781169211 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-11-12T09:26:14,867 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:26:14,867 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:26:14,868 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:26:14,868 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:26:14,868 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-11-12T09:26:14,868 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:26:14,868 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-11-12T09:26:14,869 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:26:14,873 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:59213, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-12T09:26:14,878 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1699781204878 2023-11-12T09:26:14,880 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-11-12T09:26:14,887 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43639 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3212) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:592) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-12T09:26:14,891 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-11-12T09:26:14,895 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-11-12T09:26:14,896 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-11-12T09:26:14,902 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:14,903 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-12T09:26:14,908 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-11-12T09:26:14,908 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-11-12T09:26:14,911 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-11-12T09:26:14,912 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-11-12T09:26:14,912 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-11-12T09:26:14,914 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:14,922 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-11-12T09:26:14,925 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-11-12T09:26:14,926 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-11-12T09:26:14,942 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-11-12T09:26:14,942 WARN [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-11-12T09:26:14,944 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x59126be1 to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:14,960 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63e032f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:14,960 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-12T09:26:14,963 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-11-12T09:26:14,964 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-11-12T09:26:14,966 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781174965,5,FailOnTimeoutGroup] 2023-11-12T09:26:14,966 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781174966,5,FailOnTimeoutGroup] 2023-11-12T09:26:14,966 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:14,966 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1636): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-11-12T09:26:14,970 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:14,970 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:15,043 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,43639,1699781169211 with isa=jenkins-hbase3.apache.org/172.31.12.81:36287, startcode=1699781172353 2023-11-12T09:26:15,049 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43639 {}] master.ServerManager(396): Registering regionserver=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:15,061 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:15,062 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:34955 2023-11-12T09:26:15,062 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=34843 2023-11-12T09:26:15,072 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-11-12T09:26:15,072 DEBUG [RS:0;jenkins-hbase3:36287 {}] zookeeper.ZKUtil(111): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:15,072 WARN [RS:0;jenkins-hbase3:36287 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-12T09:26:15,073 INFO [RS:0;jenkins-hbase3:36287 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:26:15,073 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:15,077 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,36287,1699781172353] 2023-11-12T09:26:15,091 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-11-12T09:26:15,107 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-11-12T09:26:15,127 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-11-12T09:26:15,134 INFO [RS:0;jenkins-hbase3:36287 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-11-12T09:26:15,134 INFO [RS:0;jenkins-hbase3:36287 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:15,136 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-11-12T09:26:15,147 INFO [RS:0;jenkins-hbase3:36287 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:15,147 DEBUG [RS:0;jenkins-hbase3:36287 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:26:15,147 DEBUG [RS:0;jenkins-hbase3:36287 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:26:15,148 DEBUG [RS:0;jenkins-hbase3:36287 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:26:15,148 DEBUG [RS:0;jenkins-hbase3:36287 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:26:15,148 DEBUG [RS:0;jenkins-hbase3:36287 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:26:15,148 DEBUG [RS:0;jenkins-hbase3:36287 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-11-12T09:26:15,148 DEBUG [RS:0;jenkins-hbase3:36287 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:26:15,148 DEBUG [RS:0;jenkins-hbase3:36287 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:26:15,148 DEBUG [RS:0;jenkins-hbase3:36287 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:26:15,149 DEBUG [RS:0;jenkins-hbase3:36287 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:26:15,149 DEBUG [RS:0;jenkins-hbase3:36287 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:26:15,149 DEBUG [RS:0;jenkins-hbase3:36287 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-11-12T09:26:15,149 DEBUG [RS:0;jenkins-hbase3:36287 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-11-12T09:26:15,150 INFO [RS:0;jenkins-hbase3:36287 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:15,150 INFO [RS:0;jenkins-hbase3:36287 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:15,150 INFO [RS:0;jenkins-hbase3:36287 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:15,151 INFO [RS:0;jenkins-hbase3:36287 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:15,151 INFO [RS:0;jenkins-hbase3:36287 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36287,1699781172353-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-12T09:26:15,171 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-11-12T09:26:15,174 INFO [RS:0;jenkins-hbase3:36287 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36287,1699781172353-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:15,191 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,36287,1699781172353 started 2023-11-12T09:26:15,191 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,36287,1699781172353, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:36287, sessionid=0x1004cf43a5e0001 2023-11-12T09:26:15,192 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-11-12T09:26:15,192 DEBUG [RS:0;jenkins-hbase3:36287 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:15,192 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,36287,1699781172353' 2023-11-12T09:26:15,192 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-11-12T09:26:15,193 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-11-12T09:26:15,193 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-11-12T09:26:15,193 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-11-12T09:26:15,193 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,36287,1699781172353' 2023-11-12T09:26:15,194 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:26:15,194 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:26:15,195 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-11-12T09:26:15,195 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-11-12T09:26:15,195 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-11-12T09:26:15,195 DEBUG [RS:0;jenkins-hbase3:36287 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:15,195 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,36287,1699781172353' 2023-11-12T09:26:15,195 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-11-12T09:26:15,195 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-11-12T09:26:15,196 DEBUG [RS:0;jenkins-hbase3:36287 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-11-12T09:26:15,196 INFO [RS:0;jenkins-hbase3:36287 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-11-12T09:26:15,196 INFO [RS:0;jenkins-hbase3:36287 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-11-12T09:26:15,307 INFO [RS:0;jenkins-hbase3:36287 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-12T09:26:15,310 INFO [RS:0;jenkins-hbase3:36287 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C36287%2C1699781172353, suffix=, logDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353, archiveDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs, maxLogs=32 2023-11-12T09:26:15,323 DEBUG [RS:0;jenkins-hbase3:36287 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781175312, exclude list is [], retry=0 2023-11-12T09:26:15,329 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:26:15,335 INFO [RS:0;jenkins-hbase3:36287 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781175312 2023-11-12T09:26:15,335 DEBUG [RS:0;jenkins-hbase3:36287 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:26:15,339 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-11-12T09:26:15,339 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:15,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:15,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-12T09:26:15,759 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-12T09:26:15,759 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:15,760 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:26:15,760 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-12T09:26:15,762 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-12T09:26:15,762 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:15,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:26:15,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-12T09:26:15,765 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-12T09:26:15,765 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:15,766 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:26:15,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740 2023-11-12T09:26:15,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740 2023-11-12T09:26:15,771 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-12T09:26:15,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-12T09:26:15,777 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:26:15,778 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11079922720, jitterRate=0.03189821541309357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-12T09:26:15,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-12T09:26:15,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-12T09:26:15,780 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-12T09:26:15,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-12T09:26:15,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-12T09:26:15,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-12T09:26:15,782 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-12T09:26:15,783 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-12T09:26:15,786 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-11-12T09:26:15,786 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-11-12T09:26:15,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-11-12T09:26:15,808 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-11-12T09:26:15,812 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-11-12T09:26:15,964 DEBUG [jenkins-hbase3:43639 {}] assignment.AssignmentManager(2319): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-11-12T09:26:15,976 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:15,982 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,36287,1699781172353, state=OPENING 2023-11-12T09:26:15,991 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-11-12T09:26:15,993 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:15,993 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:15,994 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:26:15,994 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:26:15,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:26:16,304 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:16,307 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-11-12T09:26:16,310 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:37816, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-11-12T09:26:16,323 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-11-12T09:26:16,324 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:26:16,324 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-11-12T09:26:16,328 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta, suffix=.meta, logDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353, archiveDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs, maxLogs=32 2023-11-12T09:26:16,340 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781176329.meta, exclude list is [], retry=0 2023-11-12T09:26:16,344 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:26:16,348 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781176329.meta 2023-11-12T09:26:16,348 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:26:16,348 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:26:16,353 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:26:16,354 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-11-12T09:26:16,370 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-11-12T09:26:16,370 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-11-12T09:26:16,376 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-11-12T09:26:16,376 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:16,376 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-11-12T09:26:16,376 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-11-12T09:26:16,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-12T09:26:16,381 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-12T09:26:16,381 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:16,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:26:16,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-12T09:26:16,383 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-12T09:26:16,383 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:16,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:26:16,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-12T09:26:16,385 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-12T09:26:16,385 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:16,386 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:26:16,387 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740 2023-11-12T09:26:16,390 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740 2023-11-12T09:26:16,392 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-12T09:26:16,395 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-12T09:26:16,396 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11340188000, jitterRate=0.05613730847835541}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-12T09:26:16,397 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-12T09:26:16,409 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1699781176295 2023-11-12T09:26:16,428 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-11-12T09:26:16,428 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-11-12T09:26:16,430 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:16,432 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,36287,1699781172353, state=OPEN 2023-11-12T09:26:16,434 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-12T09:26:16,434 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-12T09:26:16,435 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:26:16,435 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:26:16,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-11-12T09:26:16,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,36287,1699781172353 in 437 msec 2023-11-12T09:26:16,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-11-12T09:26:16,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 645 msec 2023-11-12T09:26:16,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.7200 sec 2023-11-12T09:26:16,455 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1064): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1699781176455, completionTime=-1 2023-11-12T09:26:16,455 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(816): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-11-12T09:26:16,455 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1645): Joining cluster... 2023-11-12T09:26:16,522 DEBUG [hconnection-0x12da4fa3-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:16,526 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:37824, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:16,542 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1657): Number of RegionServers=1 2023-11-12T09:26:16,542 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1699781236542 2023-11-12T09:26:16,542 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1699781296542 2023-11-12T09:26:16,542 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1664): Joined the cluster in 87 msec 2023-11-12T09:26:16,568 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43639,1699781169211-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:16,568 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43639,1699781169211-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:16,568 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43639,1699781169211-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:16,570 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:43639, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:16,571 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-11-12T09:26:16,577 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-11-12T09:26:16,591 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-11-12T09:26:16,593 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2369): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-12T09:26:16,602 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-11-12T09:26:16,606 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:26:16,608 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:16,609 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:26:16,628 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 79ca523521e828a10ce7dfa8176221f4, NAME => 'hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:16,639 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:16,639 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 79ca523521e828a10ce7dfa8176221f4, disabling compactions & flushes 2023-11-12T09:26:16,639 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. 2023-11-12T09:26:16,639 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. 2023-11-12T09:26:16,639 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. after waiting 0 ms 2023-11-12T09:26:16,639 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. 2023-11-12T09:26:16,640 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. 2023-11-12T09:26:16,640 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 79ca523521e828a10ce7dfa8176221f4: 2023-11-12T09:26:16,642 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:26:16,661 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1699781176644"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781176644"}]},"ts":"1699781176644"} 2023-11-12T09:26:16,706 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:26:16,708 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:26:16,717 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781176708"}]},"ts":"1699781176708"} 2023-11-12T09:26:16,724 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-11-12T09:26:16,730 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=79ca523521e828a10ce7dfa8176221f4, ASSIGN}] 2023-11-12T09:26:16,733 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=79ca523521e828a10ce7dfa8176221f4, ASSIGN 2023-11-12T09:26:16,735 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=79ca523521e828a10ce7dfa8176221f4, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:26:16,886 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=79ca523521e828a10ce7dfa8176221f4, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:16,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 79ca523521e828a10ce7dfa8176221f4, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:26:17,044 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:17,050 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. 2023-11-12T09:26:17,050 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 79ca523521e828a10ce7dfa8176221f4, NAME => 'hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:26:17,051 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:26:17,052 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 79ca523521e828a10ce7dfa8176221f4 2023-11-12T09:26:17,052 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:17,052 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 79ca523521e828a10ce7dfa8176221f4 2023-11-12T09:26:17,052 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 79ca523521e828a10ce7dfa8176221f4 2023-11-12T09:26:17,054 INFO [StoreOpener-79ca523521e828a10ce7dfa8176221f4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 79ca523521e828a10ce7dfa8176221f4 2023-11-12T09:26:17,056 INFO [StoreOpener-79ca523521e828a10ce7dfa8176221f4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79ca523521e828a10ce7dfa8176221f4 columnFamilyName info 2023-11-12T09:26:17,056 DEBUG [StoreOpener-79ca523521e828a10ce7dfa8176221f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:17,056 INFO [StoreOpener-79ca523521e828a10ce7dfa8176221f4-1 {}] regionserver.HStore(324): Store=79ca523521e828a10ce7dfa8176221f4/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:17,058 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/namespace/79ca523521e828a10ce7dfa8176221f4 2023-11-12T09:26:17,059 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/namespace/79ca523521e828a10ce7dfa8176221f4 2023-11-12T09:26:17,062 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 79ca523521e828a10ce7dfa8176221f4 2023-11-12T09:26:17,066 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/namespace/79ca523521e828a10ce7dfa8176221f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:26:17,067 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 79ca523521e828a10ce7dfa8176221f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12069892960, jitterRate=0.12409637868404388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:26:17,068 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 79ca523521e828a10ce7dfa8176221f4: 2023-11-12T09:26:17,070 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4., pid=6, masterSystemTime=1699781177044 2023-11-12T09:26:17,073 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. 2023-11-12T09:26:17,073 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. 2023-11-12T09:26:17,074 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=79ca523521e828a10ce7dfa8176221f4, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:17,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-11-12T09:26:17,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 79ca523521e828a10ce7dfa8176221f4, server=jenkins-hbase3.apache.org,36287,1699781172353 in 188 msec 2023-11-12T09:26:17,085 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-11-12T09:26:17,085 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=79ca523521e828a10ce7dfa8176221f4, ASSIGN in 351 msec 2023-11-12T09:26:17,086 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:26:17,086 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781177086"}]},"ts":"1699781177086"} 2023-11-12T09:26:17,090 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-11-12T09:26:17,094 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:26:17,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 500 msec 2023-11-12T09:26:17,106 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-11-12T09:26:17,108 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-11-12T09:26:17,108 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:17,108 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:26:17,160 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-11-12T09:26:17,181 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:26:17,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 36 msec 2023-11-12T09:26:17,196 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-11-12T09:26:17,211 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:26:17,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 18 msec 2023-11-12T09:26:17,225 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-11-12T09:26:17,228 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-11-12T09:26:17,229 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1194): Master has completed initialization 4.754sec 2023-11-12T09:26:17,231 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-11-12T09:26:17,233 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-11-12T09:26:17,234 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-11-12T09:26:17,235 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-11-12T09:26:17,235 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-11-12T09:26:17,237 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43639,1699781169211-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-12T09:26:17,238 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43639,1699781169211-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-11-12T09:26:17,261 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1297): Balancer post startup initialization complete, took 0 seconds 2023-11-12T09:26:17,263 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-11-12T09:26:17,326 DEBUG [Listener at localhost/39325 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1176545f to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:17,330 DEBUG [Listener at localhost/39325 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39fcbc93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:17,335 DEBUG [Listener at localhost/39325 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-11-12T09:26:17,335 DEBUG [Listener at localhost/39325 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-11-12T09:26:17,349 DEBUG [hconnection-0x21d4d41-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:17,360 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:37836, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:17,371 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,43639,1699781169211 2023-11-12T09:26:17,371 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-11-12T09:26:17,372 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/test.cache.data in system properties and HBase conf 2023-11-12T09:26:17,372 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop.tmp.dir in system properties and HBase conf 2023-11-12T09:26:17,372 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop.log.dir in system properties and HBase conf 2023-11-12T09:26:17,372 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-12T09:26:17,372 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-12T09:26:17,372 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-12T09:26:17,372 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:26:17,372 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:26:17,372 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-12T09:26:17,372 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:26:17,373 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-12T09:26:17,373 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-12T09:26:17,373 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:26:17,373 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:26:17,373 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-12T09:26:17,373 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/nfs.dump.dir in system properties and HBase conf 2023-11-12T09:26:17,373 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/java.io.tmpdir in system properties and HBase conf 2023-11-12T09:26:17,373 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:26:17,373 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-12T09:26:17,374 INFO [Listener at localhost/39325 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-11-12T09:26:19,276 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:26:19,664 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:26:19,670 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_34813_jobhistory____.425zj1/webapp 2023-11-12T09:26:19,842 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-11-12T09:26:19,842 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-11-12T09:26:19,842 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:26:19,846 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:26:19,933 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:26:20,057 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:26:20,336 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-11-12T09:26:20,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-11-12T09:26:20,349 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-11-12T09:26:20,358 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-11-12T09:26:20,358 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-11-12T09:26:20,361 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:26:20,361 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-11-12T09:26:20,361 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-12T09:26:20,361 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-11-12T09:26:20,361 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-11-12T09:26:20,361 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-11-12T09:26:20,369 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34813 2023-11-12T09:26:20,770 WARN [Listener at jenkins-hbase3.apache.org/37289 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:26:20,850 WARN [Listener at jenkins-hbase3.apache.org/37289 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:26:20,851 INFO [Listener at jenkins-hbase3.apache.org/37289 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:26:20,857 INFO [Listener at jenkins-hbase3.apache.org/37289 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_40419_cluster____7btolf/webapp 2023-11-12T09:26:20,932 INFO [Listener at jenkins-hbase3.apache.org/37289 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:26:20,932 INFO [Listener at jenkins-hbase3.apache.org/37289 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-11-12T09:26:20,932 INFO [Listener at jenkins-hbase3.apache.org/37289 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:26:20,933 INFO [Listener at jenkins-hbase3.apache.org/37289 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:26:20,941 INFO [Listener at jenkins-hbase3.apache.org/37289 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:26:21,099 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-11-12T09:26:21,101 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-11-12T09:26:21,144 INFO [Listener at jenkins-hbase3.apache.org/37289 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:26:21,257 INFO [Listener at jenkins-hbase3.apache.org/37289 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:26:21,270 INFO [Listener at jenkins-hbase3.apache.org/37289 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:40419 2023-11-12T09:26:21,829 WARN [Listener at jenkins-hbase3.apache.org/38381 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:26:21,853 WARN [Listener at jenkins-hbase3.apache.org/38381 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:26:21,853 INFO [Listener at jenkins-hbase3.apache.org/38381 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:26:21,860 INFO [Listener at jenkins-hbase3.apache.org/38381 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_39679_node____qhvzbl/webapp 2023-11-12T09:26:21,912 INFO [Listener at jenkins-hbase3.apache.org/38381 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-12T09:26:21,912 INFO [Listener at jenkins-hbase3.apache.org/38381 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:26:21,912 INFO [Listener at jenkins-hbase3.apache.org/38381 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:26:21,912 INFO [Listener at jenkins-hbase3.apache.org/38381 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:26:21,920 INFO [Listener at jenkins-hbase3.apache.org/38381 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:26:21,943 INFO [Listener at jenkins-hbase3.apache.org/38381 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:26:21,964 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:26:22,046 INFO [Listener at jenkins-hbase3.apache.org/38381 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:26:22,059 INFO [Listener at jenkins-hbase3.apache.org/38381 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:39679 2023-11-12T09:26:22,212 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:26:22,221 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:26:22,221 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:26:22,231 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_38001_node____.wnpq9h/webapp 2023-11-12T09:26:22,283 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-12T09:26:22,284 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:26:22,284 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:26:22,284 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:26:22,294 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:26:22,313 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:26:22,378 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:26:22,385 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:38001 2023-11-12T09:26:22,417 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-11-12T09:26:22,417 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:34955/backupUT 2023-11-12T09:26:22,429 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:26:22,434 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:56588, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:26:22,444 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-11-12T09:26:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-11-12T09:26:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-12T09:26:22,475 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:26:22,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 36 msec 2023-11-12T09:26:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-12T09:26:22,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-11-12T09:26:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-11-12T09:26:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-12T09:26:22,585 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:26:22,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 21 msec 2023-11-12T09:26:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-12T09:26:22,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-11-12T09:26:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-11-12T09:26:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-12T09:26:22,696 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:26:22,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 19 msec 2023-11-12T09:26:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-12T09:26:22,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-11-12T09:26:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-11-12T09:26:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-12T09:26:22,806 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:26:22,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 15 msec 2023-11-12T09:26:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-12T09:26:22,915 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'test-1699781182417', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:26:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1699781182417 2023-11-12T09:26:22,920 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1699781182417 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:26:22,920 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:22,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1699781182417" procId is: 13 2023-11-12T09:26:22,922 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1699781182417 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:26:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:26:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:26:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:26:23,354 INFO [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 5f328dc1320f3e032d0f665daf492b24, NAME => 'test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1699781182417', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:26:23,772 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(888): Instantiated test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:23,772 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1668): Closing 5f328dc1320f3e032d0f665daf492b24, disabling compactions & flushes 2023-11-12T09:26:23,773 INFO [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1690): Closing region test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. 2023-11-12T09:26:23,773 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. 2023-11-12T09:26:23,773 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. after waiting 0 ms 2023-11-12T09:26:23,773 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. 2023-11-12T09:26:23,773 INFO [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1909): Closed test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. 2023-11-12T09:26:23,773 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1622): Region close journal for 5f328dc1320f3e032d0f665daf492b24: 2023-11-12T09:26:23,774 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1699781182417 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:26:23,775 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1699781183774"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781183774"}]},"ts":"1699781183774"} 2023-11-12T09:26:23,778 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:26:23,779 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1699781182417 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:26:23,779 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781182417","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781183779"}]},"ts":"1699781183779"} 2023-11-12T09:26:23,782 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781182417, state=ENABLING in hbase:meta 2023-11-12T09:26:23,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699781182417, region=5f328dc1320f3e032d0f665daf492b24, ASSIGN}] 2023-11-12T09:26:23,791 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699781182417, region=5f328dc1320f3e032d0f665daf492b24, ASSIGN 2023-11-12T09:26:23,793 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1699781182417, region=5f328dc1320f3e032d0f665daf492b24, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:26:23,944 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=5f328dc1320f3e032d0f665daf492b24, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:23,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 5f328dc1320f3e032d0f665daf492b24, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:26:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:26:24,101 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:24,108 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. 2023-11-12T09:26:24,108 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 5f328dc1320f3e032d0f665daf492b24, NAME => 'test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:26:24,109 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:26:24,109 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1699781182417 5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:26:24,109 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:24,109 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:26:24,109 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:26:24,112 INFO [StoreOpener-5f328dc1320f3e032d0f665daf492b24-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:26:24,113 INFO [StoreOpener-5f328dc1320f3e032d0f665daf492b24-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f328dc1320f3e032d0f665daf492b24 columnFamilyName f 2023-11-12T09:26:24,114 DEBUG [StoreOpener-5f328dc1320f3e032d0f665daf492b24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:24,115 INFO [StoreOpener-5f328dc1320f3e032d0f665daf492b24-1 {}] regionserver.HStore(324): Store=5f328dc1320f3e032d0f665daf492b24/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:24,116 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:26:24,117 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:26:24,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:26:24,130 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:26:24,130 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 5f328dc1320f3e032d0f665daf492b24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9724901440, jitterRate=-0.09429797530174255}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:26:24,131 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 5f328dc1320f3e032d0f665daf492b24: 2023-11-12T09:26:24,133 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24., pid=15, masterSystemTime=1699781184101 2023-11-12T09:26:24,135 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. 2023-11-12T09:26:24,135 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. 2023-11-12T09:26:24,136 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=5f328dc1320f3e032d0f665daf492b24, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:24,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-11-12T09:26:24,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 5f328dc1320f3e032d0f665daf492b24, server=jenkins-hbase3.apache.org,36287,1699781172353 in 191 msec 2023-11-12T09:26:24,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-11-12T09:26:24,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1699781182417, region=5f328dc1320f3e032d0f665daf492b24, ASSIGN in 354 msec 2023-11-12T09:26:24,145 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1699781182417 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:26:24,146 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781182417","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781184145"}]},"ts":"1699781184145"} 2023-11-12T09:26:24,148 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781182417, state=ENABLED in hbase:meta 2023-11-12T09:26:24,151 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1699781182417 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:26:24,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1699781182417 in 1.2360 sec 2023-11-12T09:26:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:26:25,038 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: default:test-1699781182417, procId: 13 completed 2023-11-12T09:26:25,040 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x55722d8c to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:25,046 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67eb8cdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:25,049 DEBUG [hconnection-0x1bacd643-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:25,052 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:37848, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:25,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegion(8210): writing data to region test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. with WAL disabled. Data may be lost in the event of a crash. 2023-11-12T09:26:25,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns2:test-16997811824171', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:26:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16997811824171 2023-11-12T09:26:25,170 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16997811824171 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:26:25,170 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:25,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16997811824171" procId is: 16 2023-11-12T09:26:25,171 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16997811824171 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:26:25,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:26:25,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:26:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:26:25,590 INFO [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => e1e466fdb640b701a3363f445eefa78b, NAME => 'ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16997811824171', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:26:26,008 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:26,009 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1668): Closing e1e466fdb640b701a3363f445eefa78b, disabling compactions & flushes 2023-11-12T09:26:26,009 INFO [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. 2023-11-12T09:26:26,009 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. 2023-11-12T09:26:26,009 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. after waiting 0 ms 2023-11-12T09:26:26,009 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. 2023-11-12T09:26:26,009 INFO [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. 2023-11-12T09:26:26,009 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1622): Region close journal for e1e466fdb640b701a3363f445eefa78b: 2023-11-12T09:26:26,010 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16997811824171 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:26:26,011 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781186011"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781186011"}]},"ts":"1699781186011"} 2023-11-12T09:26:26,013 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:26:26,014 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16997811824171 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:26:26,015 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997811824171","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781186014"}]},"ts":"1699781186014"} 2023-11-12T09:26:26,016 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997811824171, state=ENABLING in hbase:meta 2023-11-12T09:26:26,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997811824171, region=e1e466fdb640b701a3363f445eefa78b, ASSIGN}] 2023-11-12T09:26:26,024 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997811824171, region=e1e466fdb640b701a3363f445eefa78b, ASSIGN 2023-11-12T09:26:26,025 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16997811824171, region=e1e466fdb640b701a3363f445eefa78b, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:26:26,178 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=e1e466fdb640b701a3363f445eefa78b, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:26,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure e1e466fdb640b701a3363f445eefa78b, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:26:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:26:26,334 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:26,338 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. 2023-11-12T09:26:26,339 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => e1e466fdb640b701a3363f445eefa78b, NAME => 'ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:26:26,339 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:26:26,339 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997811824171 e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:26,339 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:26,339 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:26,339 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:26,341 INFO [StoreOpener-e1e466fdb640b701a3363f445eefa78b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:26,343 INFO [StoreOpener-e1e466fdb640b701a3363f445eefa78b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1e466fdb640b701a3363f445eefa78b columnFamilyName f 2023-11-12T09:26:26,343 DEBUG [StoreOpener-e1e466fdb640b701a3363f445eefa78b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:26,344 INFO [StoreOpener-e1e466fdb640b701a3363f445eefa78b-1 {}] regionserver.HStore(324): Store=e1e466fdb640b701a3363f445eefa78b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:26,345 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:26,346 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:26,349 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:26,355 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:26:26,356 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened e1e466fdb640b701a3363f445eefa78b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9933479520, jitterRate=-0.07487262785434723}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:26:26,357 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for e1e466fdb640b701a3363f445eefa78b: 2023-11-12T09:26:26,358 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b., pid=18, masterSystemTime=1699781186334 2023-11-12T09:26:26,360 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. 2023-11-12T09:26:26,360 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. 2023-11-12T09:26:26,361 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=e1e466fdb640b701a3363f445eefa78b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:26,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-11-12T09:26:26,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure e1e466fdb640b701a3363f445eefa78b, server=jenkins-hbase3.apache.org,36287,1699781172353 in 183 msec 2023-11-12T09:26:26,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-11-12T09:26:26,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16997811824171, region=e1e466fdb640b701a3363f445eefa78b, ASSIGN in 345 msec 2023-11-12T09:26:26,370 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16997811824171 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:26:26,370 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997811824171","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781186370"}]},"ts":"1699781186370"} 2023-11-12T09:26:26,372 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997811824171, state=ENABLED in hbase:meta 2023-11-12T09:26:26,375 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16997811824171 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:26:26,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16997811824171 in 1.2100 sec 2023-11-12T09:26:27,102 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16997811824171' 2023-11-12T09:26:27,103 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1699781182417' 2023-11-12T09:26:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:26:27,281 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns2:test-16997811824171, procId: 16 completed 2023-11-12T09:26:27,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegion(8210): writing data to region ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. with WAL disabled. Data may be lost in the event of a crash. 2023-11-12T09:26:27,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns3:test-16997811824172', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:26:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16997811824172 2023-11-12T09:26:27,379 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16997811824172 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:26:27,379 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:27,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16997811824172" procId is: 19 2023-11-12T09:26:27,381 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16997811824172 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:26:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:26:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:26:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:26:27,798 INFO [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 72ce602541d9ecd5e0168bcd44eb49bd, NAME => 'ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16997811824172', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:26:28,215 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:28,216 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1668): Closing 72ce602541d9ecd5e0168bcd44eb49bd, disabling compactions & flushes 2023-11-12T09:26:28,216 INFO [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. 2023-11-12T09:26:28,216 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. 2023-11-12T09:26:28,216 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. after waiting 0 ms 2023-11-12T09:26:28,216 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. 2023-11-12T09:26:28,216 INFO [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. 2023-11-12T09:26:28,216 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1622): Region close journal for 72ce602541d9ecd5e0168bcd44eb49bd: 2023-11-12T09:26:28,217 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16997811824172 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:26:28,218 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781188218"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781188218"}]},"ts":"1699781188218"} 2023-11-12T09:26:28,220 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:26:28,221 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16997811824172 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:26:28,222 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997811824172","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781188221"}]},"ts":"1699781188221"} 2023-11-12T09:26:28,223 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997811824172, state=ENABLING in hbase:meta 2023-11-12T09:26:28,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997811824172, region=72ce602541d9ecd5e0168bcd44eb49bd, ASSIGN}] 2023-11-12T09:26:28,233 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997811824172, region=72ce602541d9ecd5e0168bcd44eb49bd, ASSIGN 2023-11-12T09:26:28,234 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16997811824172, region=72ce602541d9ecd5e0168bcd44eb49bd, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:26:28,386 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=72ce602541d9ecd5e0168bcd44eb49bd, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:28,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 72ce602541d9ecd5e0168bcd44eb49bd, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:26:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:26:28,541 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:28,546 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. 2023-11-12T09:26:28,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 72ce602541d9ecd5e0168bcd44eb49bd, NAME => 'ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:26:28,546 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:26:28,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997811824172 72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:28,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:28,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:28,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:28,548 INFO [StoreOpener-72ce602541d9ecd5e0168bcd44eb49bd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:28,550 INFO [StoreOpener-72ce602541d9ecd5e0168bcd44eb49bd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72ce602541d9ecd5e0168bcd44eb49bd columnFamilyName f 2023-11-12T09:26:28,550 DEBUG [StoreOpener-72ce602541d9ecd5e0168bcd44eb49bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:28,551 INFO [StoreOpener-72ce602541d9ecd5e0168bcd44eb49bd-1 {}] regionserver.HStore(324): Store=72ce602541d9ecd5e0168bcd44eb49bd/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:28,552 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:28,553 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:28,555 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:28,558 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/72ce602541d9ecd5e0168bcd44eb49bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:26:28,559 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 72ce602541d9ecd5e0168bcd44eb49bd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11095137600, jitterRate=0.03331521153450012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:26:28,560 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 72ce602541d9ecd5e0168bcd44eb49bd: 2023-11-12T09:26:28,561 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd., pid=21, masterSystemTime=1699781188541 2023-11-12T09:26:28,564 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. 2023-11-12T09:26:28,564 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. 2023-11-12T09:26:28,565 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=72ce602541d9ecd5e0168bcd44eb49bd, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:28,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-11-12T09:26:28,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 72ce602541d9ecd5e0168bcd44eb49bd, server=jenkins-hbase3.apache.org,36287,1699781172353 in 179 msec 2023-11-12T09:26:28,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-11-12T09:26:28,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16997811824172, region=72ce602541d9ecd5e0168bcd44eb49bd, ASSIGN in 340 msec 2023-11-12T09:26:28,575 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16997811824172 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:26:28,575 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997811824172","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781188575"}]},"ts":"1699781188575"} 2023-11-12T09:26:28,577 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997811824172, state=ENABLED in hbase:meta 2023-11-12T09:26:28,585 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16997811824172 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:26:28,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16997811824172 in 1.2100 sec 2023-11-12T09:26:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:26:29,490 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns3:test-16997811824172, procId: 19 completed 2023-11-12T09:26:29,490 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16997811824172 get assigned. Timeout = 60000ms 2023-11-12T09:26:29,493 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:26:29,497 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16997811824172 assigned to meta. Checking AM states. 2023-11-12T09:26:29,498 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:26:29,498 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16997811824172 assigned. 2023-11-12T09:26:29,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns4:test-16997811824173', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:26:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16997811824173 2023-11-12T09:26:29,504 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16997811824173 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:26:29,504 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:29,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16997811824173" procId is: 22 2023-11-12T09:26:29,505 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16997811824173 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:26:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:26:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:26:29,615 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:26:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:26:29,919 INFO [RegionOpenAndInit-ns4:test-16997811824173-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d4d2b2f12334f3e59195d63ae19e1266, NAME => 'ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16997811824173', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:26:30,333 DEBUG [RegionOpenAndInit-ns4:test-16997811824173-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:30,333 DEBUG [RegionOpenAndInit-ns4:test-16997811824173-pool-0 {}] regionserver.HRegion(1668): Closing d4d2b2f12334f3e59195d63ae19e1266, disabling compactions & flushes 2023-11-12T09:26:30,333 INFO [RegionOpenAndInit-ns4:test-16997811824173-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. 2023-11-12T09:26:30,333 DEBUG [RegionOpenAndInit-ns4:test-16997811824173-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. 2023-11-12T09:26:30,333 DEBUG [RegionOpenAndInit-ns4:test-16997811824173-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. after waiting 0 ms 2023-11-12T09:26:30,333 DEBUG [RegionOpenAndInit-ns4:test-16997811824173-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. 2023-11-12T09:26:30,333 INFO [RegionOpenAndInit-ns4:test-16997811824173-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. 2023-11-12T09:26:30,333 DEBUG [RegionOpenAndInit-ns4:test-16997811824173-pool-0 {}] regionserver.HRegion(1622): Region close journal for d4d2b2f12334f3e59195d63ae19e1266: 2023-11-12T09:26:30,335 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16997811824173 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:26:30,335 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781190335"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781190335"}]},"ts":"1699781190335"} 2023-11-12T09:26:30,337 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:26:30,339 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16997811824173 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:26:30,339 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16997811824173","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781190339"}]},"ts":"1699781190339"} 2023-11-12T09:26:30,341 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16997811824173, state=ENABLING in hbase:meta 2023-11-12T09:26:30,347 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997811824171 2023-11-12T09:26:30,347 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997811824171 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:26:30,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16997811824173, region=d4d2b2f12334f3e59195d63ae19e1266, ASSIGN}] 2023-11-12T09:26:30,349 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16997811824172 2023-11-12T09:26:30,349 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16997811824172 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:26:30,350 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16997811824173, region=d4d2b2f12334f3e59195d63ae19e1266, ASSIGN 2023-11-12T09:26:30,351 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1699781182417 2023-11-12T09:26:30,351 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1699781182417 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:26:30,351 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16997811824173, region=d4d2b2f12334f3e59195d63ae19e1266, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:26:30,503 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=d4d2b2f12334f3e59195d63ae19e1266, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:30,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure d4d2b2f12334f3e59195d63ae19e1266, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:26:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:26:30,658 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:30,663 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. 2023-11-12T09:26:30,663 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => d4d2b2f12334f3e59195d63ae19e1266, NAME => 'ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:26:30,663 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:26:30,663 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997811824173 d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:26:30,663 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:30,664 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:26:30,664 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:26:30,666 INFO [StoreOpener-d4d2b2f12334f3e59195d63ae19e1266-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:26:30,668 INFO [StoreOpener-d4d2b2f12334f3e59195d63ae19e1266-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4d2b2f12334f3e59195d63ae19e1266 columnFamilyName f 2023-11-12T09:26:30,668 DEBUG [StoreOpener-d4d2b2f12334f3e59195d63ae19e1266-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:30,668 INFO [StoreOpener-d4d2b2f12334f3e59195d63ae19e1266-1 {}] regionserver.HStore(324): Store=d4d2b2f12334f3e59195d63ae19e1266/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:30,669 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns4/test-16997811824173/d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:26:30,670 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns4/test-16997811824173/d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:26:30,673 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:26:30,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns4/test-16997811824173/d4d2b2f12334f3e59195d63ae19e1266/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:26:30,677 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened d4d2b2f12334f3e59195d63ae19e1266; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11490374720, jitterRate=0.0701245367527008}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:26:30,677 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for d4d2b2f12334f3e59195d63ae19e1266: 2023-11-12T09:26:30,678 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266., pid=24, masterSystemTime=1699781190658 2023-11-12T09:26:30,681 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. 2023-11-12T09:26:30,681 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. 2023-11-12T09:26:30,681 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=d4d2b2f12334f3e59195d63ae19e1266, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:30,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-11-12T09:26:30,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure d4d2b2f12334f3e59195d63ae19e1266, server=jenkins-hbase3.apache.org,36287,1699781172353 in 178 msec 2023-11-12T09:26:30,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-11-12T09:26:30,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16997811824173, region=d4d2b2f12334f3e59195d63ae19e1266, ASSIGN in 340 msec 2023-11-12T09:26:30,690 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16997811824173 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:26:30,690 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16997811824173","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781190690"}]},"ts":"1699781190690"} 2023-11-12T09:26:30,692 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16997811824173, state=ENABLED in hbase:meta 2023-11-12T09:26:30,695 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16997811824173 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:26:30,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16997811824173 in 1.1950 sec 2023-11-12T09:26:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:26:31,615 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns4:test-16997811824173, procId: 22 completed 2023-11-12T09:26:31,615 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16997811824173 get assigned. Timeout = 60000ms 2023-11-12T09:26:31,615 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:26:31,619 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16997811824173 assigned to meta. Checking AM states. 2023-11-12T09:26:31,619 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:26:31,619 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16997811824173 assigned. 2023-11-12T09:26:31,619 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x55722d8c to 127.0.0.1:56042 2023-11-12T09:26:31,619 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:26:31,663 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleOverwrite Thread=585, OpenFileDescriptor=689, MaxFileDescriptor=60000, SystemLoadAverage=212, ProcessCount=168, AvailableMemoryMB=5496 2023-11-12T09:26:31,663 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=585 is superior to 500 2023-11-12T09:26:31,666 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(216): create full backup image on multiple tables 2023-11-12T09:26:31,667 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x61c9738d to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:31,673 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f798b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:31,682 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:31,683 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54328, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:31,696 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:26:31,698 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:44974, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:26:31,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-11-12T09:26:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-11-12T09:26:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-12T09:26:31,719 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:26:31,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 16 msec 2023-11-12T09:26:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-12T09:26:31,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:26:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-11-12T09:26:31,834 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:26:31,834 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:31,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-11-12T09:26:31,835 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:26:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:26:31,852 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 606338a328e82acae51355d485ed81f0, NAME => 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:31,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:26:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:26:32,263 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:32,263 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 606338a328e82acae51355d485ed81f0, disabling compactions & flushes 2023-11-12T09:26:32,263 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:26:32,263 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:26:32,263 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. after waiting 0 ms 2023-11-12T09:26:32,263 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:26:32,263 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:26:32,264 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:26:32,265 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:26:32,266 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1699781192266"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781192266"}]},"ts":"1699781192266"} 2023-11-12T09:26:32,268 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:26:32,269 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:26:32,269 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781192269"}]},"ts":"1699781192269"} 2023-11-12T09:26:32,271 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-12T09:26:32,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=606338a328e82acae51355d485ed81f0, ASSIGN}] 2023-11-12T09:26:32,280 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=606338a328e82acae51355d485ed81f0, ASSIGN 2023-11-12T09:26:32,281 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=606338a328e82acae51355d485ed81f0, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:26:32,433 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=606338a328e82acae51355d485ed81f0, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:32,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 606338a328e82acae51355d485ed81f0, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:26:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:26:32,588 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:32,593 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:26:32,593 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 606338a328e82acae51355d485ed81f0, NAME => 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:26:32,594 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:26:32,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 606338a328e82acae51355d485ed81f0 2023-11-12T09:26:32,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:32,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 606338a328e82acae51355d485ed81f0 2023-11-12T09:26:32,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 606338a328e82acae51355d485ed81f0 2023-11-12T09:26:32,595 INFO [StoreOpener-606338a328e82acae51355d485ed81f0-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 606338a328e82acae51355d485ed81f0 2023-11-12T09:26:32,598 INFO [StoreOpener-606338a328e82acae51355d485ed81f0-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 606338a328e82acae51355d485ed81f0 columnFamilyName meta 2023-11-12T09:26:32,598 DEBUG [StoreOpener-606338a328e82acae51355d485ed81f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:32,599 INFO [StoreOpener-606338a328e82acae51355d485ed81f0-1 {}] regionserver.HStore(324): Store=606338a328e82acae51355d485ed81f0/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:32,599 INFO [StoreOpener-606338a328e82acae51355d485ed81f0-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 606338a328e82acae51355d485ed81f0 2023-11-12T09:26:32,601 INFO [StoreOpener-606338a328e82acae51355d485ed81f0-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 606338a328e82acae51355d485ed81f0 columnFamilyName session 2023-11-12T09:26:32,601 DEBUG [StoreOpener-606338a328e82acae51355d485ed81f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:32,601 INFO [StoreOpener-606338a328e82acae51355d485ed81f0-1 {}] regionserver.HStore(324): Store=606338a328e82acae51355d485ed81f0/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:32,602 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0 2023-11-12T09:26:32,603 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0 2023-11-12T09:26:32,604 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:26:32,606 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 606338a328e82acae51355d485ed81f0 2023-11-12T09:26:32,608 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:26:32,609 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 606338a328e82acae51355d485ed81f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9590277600, jitterRate=-0.10683579742908478}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:26:32,610 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:26:32,611 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1699781191829.606338a328e82acae51355d485ed81f0., pid=28, masterSystemTime=1699781192588 2023-11-12T09:26:32,613 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:26:32,613 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:26:32,613 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=606338a328e82acae51355d485ed81f0, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:32,617 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-11-12T09:26:32,617 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 606338a328e82acae51355d485ed81f0, server=jenkins-hbase3.apache.org,36287,1699781172353 in 180 msec 2023-11-12T09:26:32,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-11-12T09:26:32,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=606338a328e82acae51355d485ed81f0, ASSIGN in 340 msec 2023-11-12T09:26:32,620 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:26:32,620 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781192620"}]},"ts":"1699781192620"} 2023-11-12T09:26:32,622 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-12T09:26:32,625 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:26:32,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 796 msec 2023-11-12T09:26:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:26:32,941 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-11-12T09:26:32,956 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:26:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-11-12T09:26:32,960 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:26:32,960 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:32,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-11-12T09:26:32,961 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:26:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:26:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:26:33,104 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16997811824172' 2023-11-12T09:26:33,104 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16997811824173' 2023-11-12T09:26:33,105 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-12T09:26:33,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:26:33,375 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9276b632fc4549b20f67f1151f8dc56f, NAME => 'backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:26:33,796 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:33,796 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 9276b632fc4549b20f67f1151f8dc56f, disabling compactions & flushes 2023-11-12T09:26:33,796 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. 2023-11-12T09:26:33,796 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. 2023-11-12T09:26:33,796 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. after waiting 0 ms 2023-11-12T09:26:33,797 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. 2023-11-12T09:26:33,797 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. 2023-11-12T09:26:33,797 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9276b632fc4549b20f67f1151f8dc56f: 2023-11-12T09:26:33,798 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:26:33,798 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1699781193798"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781193798"}]},"ts":"1699781193798"} 2023-11-12T09:26:33,801 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:26:33,802 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:26:33,802 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781193802"}]},"ts":"1699781193802"} 2023-11-12T09:26:33,804 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-11-12T09:26:33,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=9276b632fc4549b20f67f1151f8dc56f, ASSIGN}] 2023-11-12T09:26:33,813 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=9276b632fc4549b20f67f1151f8dc56f, ASSIGN 2023-11-12T09:26:33,814 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=9276b632fc4549b20f67f1151f8dc56f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:26:33,965 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=9276b632fc4549b20f67f1151f8dc56f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:33,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 9276b632fc4549b20f67f1151f8dc56f, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:26:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:26:34,120 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:34,124 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. 2023-11-12T09:26:34,124 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 9276b632fc4549b20f67f1151f8dc56f, NAME => 'backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:26:34,125 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:26:34,125 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 9276b632fc4549b20f67f1151f8dc56f 2023-11-12T09:26:34,125 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:34,125 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 9276b632fc4549b20f67f1151f8dc56f 2023-11-12T09:26:34,125 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 9276b632fc4549b20f67f1151f8dc56f 2023-11-12T09:26:34,127 INFO [StoreOpener-9276b632fc4549b20f67f1151f8dc56f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 9276b632fc4549b20f67f1151f8dc56f 2023-11-12T09:26:34,129 INFO [StoreOpener-9276b632fc4549b20f67f1151f8dc56f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9276b632fc4549b20f67f1151f8dc56f columnFamilyName meta 2023-11-12T09:26:34,129 DEBUG [StoreOpener-9276b632fc4549b20f67f1151f8dc56f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:34,130 INFO [StoreOpener-9276b632fc4549b20f67f1151f8dc56f-1 {}] regionserver.HStore(324): Store=9276b632fc4549b20f67f1151f8dc56f/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:34,130 INFO [StoreOpener-9276b632fc4549b20f67f1151f8dc56f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 9276b632fc4549b20f67f1151f8dc56f 2023-11-12T09:26:34,132 INFO [StoreOpener-9276b632fc4549b20f67f1151f8dc56f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9276b632fc4549b20f67f1151f8dc56f columnFamilyName session 2023-11-12T09:26:34,132 DEBUG [StoreOpener-9276b632fc4549b20f67f1151f8dc56f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:34,133 INFO [StoreOpener-9276b632fc4549b20f67f1151f8dc56f-1 {}] regionserver.HStore(324): Store=9276b632fc4549b20f67f1151f8dc56f/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:34,134 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system_bulk/9276b632fc4549b20f67f1151f8dc56f 2023-11-12T09:26:34,134 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system_bulk/9276b632fc4549b20f67f1151f8dc56f 2023-11-12T09:26:34,136 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:26:34,138 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 9276b632fc4549b20f67f1151f8dc56f 2023-11-12T09:26:34,140 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system_bulk/9276b632fc4549b20f67f1151f8dc56f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:26:34,141 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 9276b632fc4549b20f67f1151f8dc56f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10208302880, jitterRate=-0.04927770793437958}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:26:34,141 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 9276b632fc4549b20f67f1151f8dc56f: 2023-11-12T09:26:34,142 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f., pid=31, masterSystemTime=1699781194119 2023-11-12T09:26:34,145 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. 2023-11-12T09:26:34,145 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. 2023-11-12T09:26:34,145 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=9276b632fc4549b20f67f1151f8dc56f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:34,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-11-12T09:26:34,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 9276b632fc4549b20f67f1151f8dc56f, server=jenkins-hbase3.apache.org,36287,1699781172353 in 180 msec 2023-11-12T09:26:34,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-11-12T09:26:34,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=9276b632fc4549b20f67f1151f8dc56f, ASSIGN in 339 msec 2023-11-12T09:26:34,152 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:26:34,152 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781194152"}]},"ts":"1699781194152"} 2023-11-12T09:26:34,154 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-11-12T09:26:34,157 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:26:34,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.2010 sec 2023-11-12T09:26:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:26:35,069 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-11-12T09:26:35,082 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:26:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2023-11-12T09:26:35,117 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:26:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781195117 (current time:1699781195117). 2023-11-12T09:26:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:26:35,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:26:35,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x04542b37 to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:35,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ca70c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:35,131 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:56074, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x04542b37 to 127.0.0.1:56042 2023-11-12T09:26:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:26:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:26:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:26:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-12T09:26:35,149 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:26:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:26:35,155 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:26:35,168 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:26:35,170 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:26:35,182 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:26:35,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0}] 2023-11-12T09:26:35,193 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:26:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:26:35,353 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:35,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-11-12T09:26:35,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:26:35,358 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=45 B heapSize=632 B 2023-11-12T09:26:35,440 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/033eb5dea61d4d04a65e5f0f5582e1e8 is 49, key is activesession:/session:c/1699781195103/Put/seqid=0 2023-11-12T09:26:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:26:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:26:35,860 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/033eb5dea61d4d04a65e5f0f5582e1e8 2023-11-12T09:26:35,948 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/033eb5dea61d4d04a65e5f0f5582e1e8 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 2023-11-12T09:26:35,958 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8, entries=1, sequenceid=5, filesize=4.9 K 2023-11-12T09:26:35,963 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 605ms, sequenceid=5, compaction requested=false 2023-11-12T09:26:35,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:26:35,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. for snapshot_backup_system completed. 2023-11-12T09:26:35,967 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:26:35,973 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:26:35,978 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-12T09:26:35,978 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8] hfiles 2023-11-12T09:26:35,978 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 for snapshot=snapshot_backup_system 2023-11-12T09:26:35,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:26:36,003 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-11-12T09:26:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=33 2023-11-12T09:26:36,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 606338a328e82acae51355d485ed81f0 2023-11-12T09:26:36,007 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:26:36,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-11-12T09:26:36,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 in 824 msec 2023-11-12T09:26:36,011 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:26:36,013 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:26:36,016 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:26:36,016 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:26:36,019 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:26:36,048 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:26:36,063 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:26:36,064 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:26:36,066 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:26:36,067 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-12T09:26:36,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 926 msec 2023-11-12T09:26:36,169 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:26:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:26:36,257 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-11-12T09:26:36,257 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(115): Backup backup_1699781191676 started at 1699781196257. 2023-11-12T09:26:36,280 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(119): Backup session backup_1699781191676 has been started. 2023-11-12T09:26:36,283 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:26:36,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:26:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:26:36,302 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:26:36,302 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:26:36,303 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:26:36,303 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:26:36,304 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:26:36,304 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:26:36,306 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,306 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:26:36,306 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:26:36,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:26:36,307 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,307 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:26:36,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:26:36,307 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:26:36,308 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:26:36,308 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:26:36,308 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:26:36,310 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:26:36,311 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:26:36,311 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:26:36,311 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:26:36,312 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:26:36,312 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:26:36,312 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:26:36,314 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,314 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:36,315 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,315 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:26:36,315 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:26:36,315 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:26:36,315 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:26:36,315 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:26:36,316 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:26:36,316 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:26:36,316 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,317 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:26:36,317 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:26:36,317 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5b235f17[Count = 0] remaining members to acquire global barrier 2023-11-12T09:26:36,317 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:26:36,317 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:36,320 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:36,320 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:36,320 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:36,320 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:26:36,320 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,320 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:26:36,321 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:26:36,321 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,321 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781175312 highest: 1699781175312 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,322 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781175312) roll requested 2023-11-12T09:26:36,333 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781196322, exclude list is [], retry=0 2023-11-12T09:26:36,341 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:26:36,345 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781175312 with entries=19, filesize=4.02 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781196322 2023-11-12T09:26:36,346 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:26:36,346 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781175312 is not closed yet, will try archiving it next time 2023-11-12T09:26:36,346 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781176329) roll requested 2023-11-12T09:26:36,358 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781196346.meta, exclude list is [], retry=0 2023-11-12T09:26:36,362 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:26:36,362 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:26:36,362 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781196322 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,363 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:26:36,365 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35616, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:26:36,366 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781176329.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781196346.meta 2023-11-12T09:26:36,368 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:26:36,368 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781176329.meta is not closed yet, will try archiving it next time 2023-11-12T09:26:36,387 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:26:36,387 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36287,1699781172353' in zk 2023-11-12T09:26:36,391 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,391 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:26:36,392 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:26:36,392 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,392 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:26:36,393 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:26:36,393 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:26:36,393 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:26:36,394 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:26:36,394 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:26:36,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:26:36,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:26:36,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36287,1699781172353': 2023-11-12T09:26:36,398 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:26:36,398 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:26:36,398 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:26:36,398 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:26:36,398 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:26:36,398 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:26:36,403 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:26:36,403 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:26:36,403 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:26:36,403 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:26:36,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:26:36,405 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:26:36,406 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:26:36,406 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,406 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:26:36,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:26:36,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:26:36,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:26:36,408 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,409 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:26:36,409 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:26:36,410 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:26:36,411 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:26:36,411 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,412 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:26:36,412 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:26:36,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,423 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,423 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:26:36,423 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:26:36,423 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,423 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:36,423 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:36,423 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:26:36,423 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-11-12T09:26:36,423 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:26:36,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:26:36,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:26:36,423 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:26:36,423 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:26:36,424 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:26:36,424 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:26:36,424 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:26:36,426 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:26:36,427 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-12T09:26:36,427 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:26:36,527 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:26:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:26:36,538 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } 2023-11-12T09:26:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781196538 (current time:1699781196538). 2023-11-12T09:26:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:26:36,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781196537_ns2_test-16997811824171 VERSION not specified, setting to 2 2023-11-12T09:26:36,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x245a083d to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ff8f77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:36,550 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:56082, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x245a083d to 127.0.0.1:56042 2023-11-12T09:26:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:26:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:26:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } 2023-11-12T09:26:36,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-11-12T09:26:36,556 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:26:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:26:36,557 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:26:36,560 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781196537_ns2_test-16997811824171 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:26:36,562 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:26:36,574 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:26:36,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure e1e466fdb640b701a3363f445eefa78b}] 2023-11-12T09:26:36,575 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:26:36,727 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:36,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-11-12T09:26:36,728 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. 2023-11-12T09:26:36,730 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing e1e466fdb640b701a3363f445eefa78b 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-12T09:26:36,752 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/.tmp/f/6da1d5d983914342bd21c6506217868a is 37, key is row10/f:q1/1699781187295/Put/seqid=0 2023-11-12T09:26:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:26:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:26:37,163 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/.tmp/f/6da1d5d983914342bd21c6506217868a 2023-11-12T09:26:37,175 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/.tmp/f/6da1d5d983914342bd21c6506217868a as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/f/6da1d5d983914342bd21c6506217868a 2023-11-12T09:26:37,187 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/f/6da1d5d983914342bd21c6506217868a, entries=99, sequenceid=103, filesize=8.2 K 2023-11-12T09:26:37,189 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for e1e466fdb640b701a3363f445eefa78b in 459ms, sequenceid=103, compaction requested=false 2023-11-12T09:26:37,190 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for e1e466fdb640b701a3363f445eefa78b: 2023-11-12T09:26:37,190 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. for snapshot_1699781196537_ns2_test-16997811824171 completed. 2023-11-12T09:26:37,191 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b.' region-info for snapshot=snapshot_1699781196537_ns2_test-16997811824171 2023-11-12T09:26:37,191 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:26:37,191 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/f/6da1d5d983914342bd21c6506217868a] hfiles 2023-11-12T09:26:37,191 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/f/6da1d5d983914342bd21c6506217868a for snapshot=snapshot_1699781196537_ns2_test-16997811824171 2023-11-12T09:26:37,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. 2023-11-12T09:26:37,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-11-12T09:26:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=35 2023-11-12T09:26:37,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781196537_ns2_test-16997811824171 on region e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:37,608 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:37,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-11-12T09:26:37,612 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:26:37,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure e1e466fdb640b701a3363f445eefa78b in 1.0350 sec 2023-11-12T09:26:37,613 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:26:37,613 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:26:37,613 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781196537_ns2_test-16997811824171 2023-11-12T09:26:37,614 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781196537_ns2_test-16997811824171 2023-11-12T09:26:37,636 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:26:37,644 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:26:37,645 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781196537_ns2_test-16997811824171 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781196537_ns2_test-16997811824171 2023-11-12T09:26:37,647 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:26:37,647 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-11-12T09:26:37,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781196537_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } in 1.0950 sec 2023-11-12T09:26:37,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:26:37,664 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: ns2:test-16997811824171, procId: 34 completed 2023-11-12T09:26:37,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } 2023-11-12T09:26:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781197665 (current time:1699781197665). 2023-11-12T09:26:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:26:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781197664_ns3_test-16997811824172 VERSION not specified, setting to 2 2023-11-12T09:26:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x367585b7 to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@429a68a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:37,676 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:56090, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x367585b7 to 127.0.0.1:56042 2023-11-12T09:26:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:26:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:26:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } 2023-11-12T09:26:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-11-12T09:26:37,683 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:26:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:26:37,684 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:26:37,689 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781197664_ns3_test-16997811824172 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:26:37,692 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:26:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:26:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:26:38,106 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:26:38,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 72ce602541d9ecd5e0168bcd44eb49bd}] 2023-11-12T09:26:38,108 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:38,260 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:38,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-11-12T09:26:38,262 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. 2023-11-12T09:26:38,263 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 72ce602541d9ecd5e0168bcd44eb49bd: 2023-11-12T09:26:38,263 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. for snapshot_1699781197664_ns3_test-16997811824172 completed. 2023-11-12T09:26:38,264 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd.' region-info for snapshot=snapshot_1699781197664_ns3_test-16997811824172 2023-11-12T09:26:38,264 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:26:38,264 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-12T09:26:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:26:38,675 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. 2023-11-12T09:26:38,675 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-11-12T09:26:38,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=37 2023-11-12T09:26:38,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781197664_ns3_test-16997811824172 on region 72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:38,678 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:38,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-11-12T09:26:38,682 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:26:38,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 72ce602541d9ecd5e0168bcd44eb49bd in 573 msec 2023-11-12T09:26:38,683 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:26:38,684 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:26:38,684 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781197664_ns3_test-16997811824172 2023-11-12T09:26:38,685 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781197664_ns3_test-16997811824172 2023-11-12T09:26:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:26:39,101 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:26:39,106 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-11-12T09:26:39,112 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:26:39,113 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781197664_ns3_test-16997811824172 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781197664_ns3_test-16997811824172 2023-11-12T09:26:39,115 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:26:39,115 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-11-12T09:26:39,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781197664_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } in 1.4370 sec 2023-11-12T09:26:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:26:39,793 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: ns3:test-16997811824172, procId: 36 completed 2023-11-12T09:26:39,793 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781191676 2023-11-12T09:26:39,793 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:26:39,795 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-12T09:26:39,797 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781191676_ns3:test-16997811824172 2023-11-12T09:26:39,797 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781197664_ns3_test-16997811824172 to hdfs://localhost:34955/backupUT/backup_1699781191676/ns3/test-16997811824172/ 2023-11-12T09:26:39,808 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:26:39,840 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:39,840 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_-1587402473_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781191676/ns3/test-16997811824172, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781191676/ns3/test-16997811824172/.hbase-snapshot/.tmp/snapshot_1699781197664_ns3_test-16997811824172 2023-11-12T09:26:39,840 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781197664_ns3_test-16997811824172. 2023-11-12T09:26:39,848 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781197664_ns3_test-16997811824172 to hdfs://localhost:34955/backupUT/backup_1699781191676/ns3/test-16997811824172/.hbase-snapshot/.tmp/snapshot_1699781197664_ns3_test-16997811824172 2023-11-12T09:26:40,283 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:40,284 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:40,285 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:40,285 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:40,347 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-12T09:26:40,347 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-11-12T09:26:40,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-11-12T09:26:40,348 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-11-12T09:26:40,349 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16997811824173 2023-11-12T09:26:40,349 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16997811824173 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:26:41,277 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-145665480947321735.jar 2023-11-12T09:26:41,277 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:41,278 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:41,279 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:41,279 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:41,279 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:41,280 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:41,281 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:41,281 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:41,282 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:41,283 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:26:41,284 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:26:41,285 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:26:41,286 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:26:41,286 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:26:41,287 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:26:41,287 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:26:41,287 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:26:41,288 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:26:41,288 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:26:41,289 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:26:41,289 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:26:41,293 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:26:41,294 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:26:41,294 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:26:41,294 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:26:41,295 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:26:41,296 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:26:41,296 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:26:41,356 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:26:41,382 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781197664_ns3_test-16997811824172' hfile list 2023-11-12T09:26:41,630 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781201578/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:26:41,631 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781201578/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:26:42,703 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:26:42,704 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:26:42,713 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781197664_ns3_test-16997811824172 2023-11-12T09:26:42,713 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781197664_ns3_test-16997811824172 finished. 2023-11-12T09:26:42,713 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781191676_ns2:test-16997811824171 2023-11-12T09:26:42,713 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781196537_ns2_test-16997811824171 to hdfs://localhost:34955/backupUT/backup_1699781191676/ns2/test-16997811824171/ 2023-11-12T09:26:42,713 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:26:42,742 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:42,742 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_226395933_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781191676/ns2/test-16997811824171, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781191676/ns2/test-16997811824171/.hbase-snapshot/.tmp/snapshot_1699781196537_ns2_test-16997811824171 2023-11-12T09:26:42,742 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781196537_ns2_test-16997811824171. 2023-11-12T09:26:42,782 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781196537_ns2_test-16997811824171 to hdfs://localhost:34955/backupUT/backup_1699781191676/ns2/test-16997811824171/.hbase-snapshot/.tmp/snapshot_1699781196537_ns2_test-16997811824171 2023-11-12T09:26:43,206 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:43,207 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:43,208 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:43,208 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:44,202 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-5300508396803820065.jar 2023-11-12T09:26:44,203 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:44,204 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:44,204 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:44,205 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:44,206 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:44,206 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:44,207 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:44,208 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:44,208 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:44,210 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:26:44,211 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:26:44,211 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:26:44,212 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:26:44,213 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:26:44,214 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:26:44,214 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:26:44,215 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:26:44,216 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:26:44,216 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:26:44,217 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:26:44,218 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:26:44,218 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:26:44,219 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:26:44,220 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:26:44,221 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:26:44,221 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:26:44,222 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:26:44,223 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:26:44,260 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:26:44,280 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781196537_ns2_test-16997811824171' hfile list 2023-11-12T09:26:44,292 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-12T09:26:44,401 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781204359/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:26:44,401 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781204359/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:26:44,517 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:26:44,548 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/f/6da1d5d983914342bd21c6506217868a output=hdfs://localhost:34955/backupUT/backup_1699781191676/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/f/6da1d5d983914342bd21c6506217868a 2023-11-12T09:26:44,548 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-11-12T09:26:45,447 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:26:45,448 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:26:45,459 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781196537_ns2_test-16997811824171 2023-11-12T09:26:45,459 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781196537_ns2_test-16997811824171 finished. 2023-11-12T09:26:45,460 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): ns3:test-16997811824172 2023-11-12T09:26:45,460 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): ns2:test-16997811824171 2023-11-12T09:26:45,558 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781191676 2023-11-12T09:26:45,558 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:26:45,572 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781191676/.backup.manifest 2023-11-12T09:26:45,572 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781191676 2023-11-12T09:26:45,572 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:26:45,578 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781191676/.backup.manifest 2023-11-12T09:26:45,579 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(358): Backup backup_1699781191676 finished: type=FULL,tablelist=ns3:test-16997811824172;ns2:test-16997811824171,targetRootDir=hdfs://localhost:34955/backupUT,startts=1699781196257,completets=1699781205555,bytescopied=0 2023-11-12T09:26:45,579 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:26:45,579 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781197664_ns3_test-16997811824172 2023-11-12T09:26:45,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781197664_ns3_test-16997811824172" 2023-11-12T09:26:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781197664_ns3_test-16997811824172 2023-11-12T09:26:45,596 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781197664_ns3_test-16997811824172 for backup backup_1699781191676 succeeded. 2023-11-12T09:26:45,596 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781196537_ns2_test-16997811824171 2023-11-12T09:26:45,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781196537_ns2_test-16997811824171" 2023-11-12T09:26:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781196537_ns2_test-16997811824171 2023-11-12T09:26:45,603 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781196537_ns2_test-16997811824171 for backup backup_1699781191676 succeeded. 2023-11-12T09:26:45,604 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:26:45,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:26:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:26:45,623 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:26:45,631 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:26:45,651 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(376): Backup backup_1699781191676 completed. 2023-11-12T09:26:45,651 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:26:45,651 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x61c9738d to 127.0.0.1:56042 2023-11-12T09:26:45,651 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:26:45,683 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781191676/.backup.manifest 2023-11-12T09:26:45,687 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781191676/.backup.manifest 2023-11-12T09:26:45,688 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781191676/.backup.manifest 2023-11-12T09:26:45,690 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781191676/.backup.manifest 2023-11-12T09:26:45,697 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781191676/.backup.manifest 2023-11-12T09:26:45,700 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781191676/.backup.manifest 2023-11-12T09:26:45,700 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16997811824171' to 'ns2:test-16997811824171' from full backup image hdfs://localhost:34955/backupUT/backup_1699781191676/ns2/test-16997811824171 2023-11-12T09:26:45,712 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(476): Truncating exising target table 'ns2:test-16997811824171', preserving region splits 2023-11-12T09:26:45,714 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$18(967): Started disable of ns2:test-16997811824171 2023-11-12T09:26:45,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable ns2:test-16997811824171 2023-11-12T09:26:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:test-16997811824171 2023-11-12T09:26:45,734 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997811824171","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781205733"}]},"ts":"1699781205733"} 2023-11-12T09:26:45,736 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997811824171, state=DISABLING in hbase:meta 2023-11-12T09:26:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:26:45,738 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set ns2:test-16997811824171 to state=DISABLING 2023-11-12T09:26:45,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16997811824171, region=e1e466fdb640b701a3363f445eefa78b, UNASSIGN}] 2023-11-12T09:26:45,742 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16997811824171, region=e1e466fdb640b701a3363f445eefa78b, UNASSIGN 2023-11-12T09:26:45,743 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=e1e466fdb640b701a3363f445eefa78b, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:45,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; CloseRegionProcedure e1e466fdb640b701a3363f445eefa78b, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:26:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:26:45,904 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:45,906 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(119): Close e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:45,906 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1668): Closing e1e466fdb640b701a3363f445eefa78b, disabling compactions & flushes 2023-11-12T09:26:45,906 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1690): Closing region ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. 2023-11-12T09:26:45,906 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. 2023-11-12T09:26:45,906 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. after waiting 0 ms 2023-11-12T09:26:45,906 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. 2023-11-12T09:26:45,915 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-11-12T09:26:45,916 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:26:45,916 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1909): Closed ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b. 2023-11-12T09:26:45,916 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1622): Region close journal for e1e466fdb640b701a3363f445eefa78b: 2023-11-12T09:26:45,919 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(163): Closed e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:45,919 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=e1e466fdb640b701a3363f445eefa78b, regionState=CLOSED 2023-11-12T09:26:45,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-11-12T09:26:45,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; CloseRegionProcedure e1e466fdb640b701a3363f445eefa78b, server=jenkins-hbase3.apache.org,36287,1699781172353 in 176 msec 2023-11-12T09:26:45,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=39, resume processing ppid=38 2023-11-12T09:26:45,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, ppid=38, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16997811824171, region=e1e466fdb640b701a3363f445eefa78b, UNASSIGN in 183 msec 2023-11-12T09:26:45,926 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997811824171","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781205926"}]},"ts":"1699781205926"} 2023-11-12T09:26:45,927 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997811824171, state=DISABLED in hbase:meta 2023-11-12T09:26:45,929 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set ns2:test-16997811824171 to state=DISABLED 2023-11-12T09:26:45,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; DisableTableProcedure table=ns2:test-16997811824171 in 207 msec 2023-11-12T09:26:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:26:46,039 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: DISABLE, Table Name: ns2:test-16997811824171, procId: 38 completed 2023-11-12T09:26:46,041 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$16(847): Started truncating ns2:test-16997811824171 2023-11-12T09:26:46,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$6(2481): Client=jenkins//172.31.12.81 truncate ns2:test-16997811824171 2023-11-12T09:26:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=41, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns2:test-16997811824171 preserveSplits=true) 2023-11-12T09:26:46,066 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:test-16997811824171' regions in transition 2023-11-12T09:26:46,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:26:46,084 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:46,089 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/f, FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/recovered.edits] 2023-11-12T09:26:46,098 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/f/6da1d5d983914342bd21c6506217868a to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/f/6da1d5d983914342bd21c6506217868a 2023-11-12T09:26:46,103 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/recovered.edits/106.seqid to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/recovered.edits/106.seqid 2023-11-12T09:26:46,103 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:46,103 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived ns2:test-16997811824171 regions 2023-11-12T09:26:46,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2023-11-12T09:26:46,118 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:test-16997811824171 from hbase:meta 2023-11-12T09:26:46,121 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:test-16997811824171' descriptor. 2023-11-12T09:26:46,121 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:test-16997811824171' from region states. 2023-11-12T09:26:46,121 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781206121"}]},"ts":"9223372036854775807"} 2023-11-12T09:26:46,124 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:26:46,124 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => e1e466fdb640b701a3363f445eefa78b, NAME => 'ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:26:46,124 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:test-16997811824171' as deleted. 2023-11-12T09:26:46,124 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16997811824171","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781206124"}]},"ts":"9223372036854775807"} 2023-11-12T09:26:46,126 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:test-16997811824171 state from META 2023-11-12T09:26:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:26:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:26:46,544 INFO [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 6e68e35ec82e3efaab2edd4a90156b95, NAME => 'ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16997811824171', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:26:46,955 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:46,955 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1668): Closing 6e68e35ec82e3efaab2edd4a90156b95, disabling compactions & flushes 2023-11-12T09:26:46,955 INFO [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:26:46,955 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:26:46,955 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. after waiting 0 ms 2023-11-12T09:26:46,955 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:26:46,955 INFO [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:26:46,955 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1622): Region close journal for 6e68e35ec82e3efaab2edd4a90156b95: 2023-11-12T09:26:46,957 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781206957"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781206957"}]},"ts":"1699781206957"} 2023-11-12T09:26:46,959 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:26:46,960 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997811824171","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781206959"}]},"ts":"1699781206959"} 2023-11-12T09:26:46,961 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997811824171, state=ENABLING in hbase:meta 2023-11-12T09:26:46,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997811824171, region=6e68e35ec82e3efaab2edd4a90156b95, ASSIGN}] 2023-11-12T09:26:46,968 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997811824171, region=6e68e35ec82e3efaab2edd4a90156b95, ASSIGN 2023-11-12T09:26:46,968 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16997811824171, region=6e68e35ec82e3efaab2edd4a90156b95, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:26:47,120 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=6e68e35ec82e3efaab2edd4a90156b95, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:47,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; OpenRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:26:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:26:47,274 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:47,279 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(130): Open ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:26:47,279 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7254): Opening region: {ENCODED => 6e68e35ec82e3efaab2edd4a90156b95, NAME => 'ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:26:47,279 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:26:47,279 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997811824171 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:26:47,279 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(888): Instantiated ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:47,279 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7296): checking encryption for 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:26:47,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7299): checking classloading for 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:26:47,281 INFO [StoreOpener-6e68e35ec82e3efaab2edd4a90156b95-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:26:47,283 INFO [StoreOpener-6e68e35ec82e3efaab2edd4a90156b95-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e68e35ec82e3efaab2edd4a90156b95 columnFamilyName f 2023-11-12T09:26:47,283 DEBUG [StoreOpener-6e68e35ec82e3efaab2edd4a90156b95-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:47,284 INFO [StoreOpener-6e68e35ec82e3efaab2edd4a90156b95-1 {}] regionserver.HStore(324): Store=6e68e35ec82e3efaab2edd4a90156b95/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:47,285 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:26:47,285 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:26:47,288 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1076): writing seq id for 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:26:47,290 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:26:47,290 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): Opened 6e68e35ec82e3efaab2edd4a90156b95; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11640222240, jitterRate=0.08408017456531525}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:26:47,291 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(992): Region open journal for 6e68e35ec82e3efaab2edd4a90156b95: 2023-11-12T09:26:47,292 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95., pid=43, masterSystemTime=1699781207274 2023-11-12T09:26:47,294 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:26:47,294 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(158): Opened ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:26:47,294 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=6e68e35ec82e3efaab2edd4a90156b95, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:47,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-11-12T09:26:47,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; OpenRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95, server=jenkins-hbase3.apache.org,36287,1699781172353 in 174 msec 2023-11-12T09:26:47,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=42, resume processing ppid=41 2023-11-12T09:26:47,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, ppid=41, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16997811824171, region=6e68e35ec82e3efaab2edd4a90156b95, ASSIGN in 332 msec 2023-11-12T09:26:47,300 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997811824171","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781207300"}]},"ts":"1699781207300"} 2023-11-12T09:26:47,302 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997811824171, state=ENABLED in hbase:meta 2023-11-12T09:26:47,308 DEBUG [PEWorker-4 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:test-16997811824171' completed 2023-11-12T09:26:47,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, state=SUCCESS; TruncateTableProcedure (table=ns2:test-16997811824171 preserveSplits=true) in 1.2520 sec 2023-11-12T09:26:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:26:48,184 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: TRUNCATE, Table Name: ns2:test-16997811824171, procId: 41 completed 2023-11-12T09:26:48,191 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34955/backupUT/backup_1699781191676/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b from hbase tables ns2:test-16997811824171 to tables ns2:test-16997811824171 2023-11-12T09:26:48,191 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16997811824171 into ns2:test-16997811824171 2023-11-12T09:26:48,194 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781208191 from hdfs://localhost:34955/backupUT/backup_1699781191676/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b 2023-11-12T09:26:48,199 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5b8ab59c to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:48,205 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4e4367, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:48,206 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:26:48,208 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54350, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:26:48,227 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:26:48,227 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16997811824171 2023-11-12T09:26:48,233 DEBUG [hconnection-0x28e669c2-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:48,234 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36018, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:48,238 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16997811824171: 2023-11-12T09:26:48,238 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-12T09:26:48,239 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/partitions_8db29319-5b31-4857-8399-e090b5569f15 2023-11-12T09:26:48,683 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:48,684 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:48,685 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:48,685 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:48,766 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-12T09:26:48,768 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54362, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-12T09:26:49,659 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-3762484694451800569.jar 2023-11-12T09:26:49,660 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,660 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,661 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,661 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,662 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,662 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,662 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,663 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,663 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,664 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:26:49,665 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:26:49,665 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:26:49,665 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:26:49,666 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:26:49,667 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:26:49,667 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:26:49,667 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:26:49,668 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:26:49,669 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:26:49,670 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:26:49,670 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:26:49,675 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,675 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,676 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,676 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,676 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,677 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:49,677 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:26:49,677 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16997811824171 2023-11-12T09:26:49,678 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:26:49,678 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5b8ab59c to 127.0.0.1:56042 2023-11-12T09:26:49,678 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:26:49,678 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-12T09:26:49,678 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-12T09:26:49,679 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:26:49,685 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:26:49,691 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-12T09:26:49,791 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781209750/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:26:49,791 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781209750/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:26:49,872 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34955/backupUT/backup_1699781191676/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/e1e466fdb640b701a3363f445eefa78b/f/6da1d5d983914342bd21c6506217868a 2023-11-12T09:26:49,880 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-12T09:26:50,007 DEBUG [pool-203-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0fd2b4ba to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:50,014 DEBUG [pool-203-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173f012, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:50,018 DEBUG [hconnection-0x65f81062-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:50,019 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36026, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:50,026 DEBUG [pool-203-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0fd2b4ba to 127.0.0.1:56042 2023-11-12T09:26:50,026 DEBUG [pool-203-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:26:50,026 DEBUG [pool-203-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-12T09:26:50,026 DEBUG [pool-203-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:26:50,043 DEBUG [pool-203-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781208191/_temporary/0/_temporary/attempt_local462771808_0003_r_000000_0/f/4848ca43e9354cf7b662cc124dc09a03 is 37, key is row10/f:q1/1699781187295/Put/seqid=0 2023-11-12T09:26:50,045 WARN [Thread-797 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36287] Specified favored nodes: [jenkins-hbase3.apache.org:36287] 2023-11-12T09:26:50,866 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781208191 2023-11-12T09:26:50,868 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0539e011 to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:50,875 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e57563e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:50,877 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:50,879 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36028, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:50,889 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781208191/_SUCCESS 2023-11-12T09:26:50,895 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:26:50,898 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54366, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:26:50,958 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781208191/f/4848ca43e9354cf7b662cc124dc09a03 first=Optional[row0] last=Optional[row98] 2023-11-12T09:26:50,989 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95., hostname=jenkins-hbase3.apache.org,36287,1699781172353, seqNum=2 for row with hfile group [{f,hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781208191/f/4848ca43e9354cf7b662cc124dc09a03}] 2023-11-12T09:26:51,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781208191/f/4848ca43e9354cf7b662cc124dc09a03 for inclusion in 6e68e35ec82e3efaab2edd4a90156b95/f 2023-11-12T09:26:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-12T09:26:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-12T09:26:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegion(2520): Flush status journal for 6e68e35ec82e3efaab2edd4a90156b95: 2023-11-12T09:26:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781208191/f/4848ca43e9354cf7b662cc124dc09a03 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_test-16997811824171__vn9rb764u2h20juc1jedgc89i6q2njsffvs7jvv3g6pqd66jo2aabsmjugm20c7j/f/4848ca43e9354cf7b662cc124dc09a03 2023-11-12T09:26:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_test-16997811824171__vn9rb764u2h20juc1jedgc89i6q2njsffvs7jvv3g6pqd66jo2aabsmjugm20c7j/f/4848ca43e9354cf7b662cc124dc09a03 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ 2023-11-12T09:26:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5e6bf502 to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b3eebb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:26:51,054 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54374, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:26:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:51,061 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36032, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(428): write bulk load descriptor to backup ns2:test-16997811824171 with 1 entries 2023-11-12T09:26:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(1602): writing raw bulk path hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ for ns2:test-16997811824171 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:26:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(435): written 1 rows for bulk load of ns2:test-16997811824171 2023-11-12T09:26:51,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:26:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5e6bf502 to 127.0.0.1:56042 2023-11-12T09:26:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:26:51,097 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_test-16997811824171__vn9rb764u2h20juc1jedgc89i6q2njsffvs7jvv3g6pqd66jo2aabsmjugm20c7j/f/4848ca43e9354cf7b662cc124dc09a03 into 6e68e35ec82e3efaab2edd4a90156b95/f as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ - updating store file list. 2023-11-12T09:26:51,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ into 6e68e35ec82e3efaab2edd4a90156b95/f 2023-11-12T09:26:51,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_test-16997811824171__vn9rb764u2h20juc1jedgc89i6q2njsffvs7jvv3g6pqd66jo2aabsmjugm20c7j/f/4848ca43e9354cf7b662cc124dc09a03 into 6e68e35ec82e3efaab2edd4a90156b95/f (new location: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_) 2023-11-12T09:26:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_test-16997811824171__vn9rb764u2h20juc1jedgc89i6q2njsffvs7jvv3g6pqd66jo2aabsmjugm20c7j/f/4848ca43e9354cf7b662cc124dc09a03 2023-11-12T09:26:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5378a185 to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75fbd07f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:26:51,130 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54378, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:26:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:51,139 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36048, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-12T09:26:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup ns2:test-16997811824171 with 1 entries 2023-11-12T09:26:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(1523): writing done bulk path hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ for ns2:test-16997811824171 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:26:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(414): written 1 rows for bulk load of ns2:test-16997811824171 2023-11-12T09:26:51,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:26:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5378a185 to 127.0.0.1:56042 2023-11-12T09:26:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:26:51,186 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:26:51,187 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0539e011 to 127.0.0.1:56042 2023-11-12T09:26:51,187 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:26:51,189 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-12T09:26:51,189 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:26:51,189 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(233): Backup: backup_1699781191676 hdfs://localhost:34955/backupUT/backup_1699781191676/ns2/test-16997811824171/ 2023-11-12T09:26:51,191 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781191676/.backup.manifest 2023-11-12T09:26:51,197 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781191676/.backup.manifest 2023-11-12T09:26:51,197 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16997811824172' to 'ns3:test-16997811824172' from full backup image hdfs://localhost:34955/backupUT/backup_1699781191676/ns3/test-16997811824172 2023-11-12T09:26:51,208 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:34955/backupUT/backup_1699781191676/ns3/test-16997811824172/archive/data/ns3/test-16997811824172 does not exists 2023-11-12T09:26:51,208 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16997811824172, will only create table 2023-11-12T09:26:51,211 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(476): Truncating exising target table 'ns3:test-16997811824172', preserving region splits 2023-11-12T09:26:51,212 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$18(967): Started disable of ns3:test-16997811824172 2023-11-12T09:26:51,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable ns3:test-16997811824172 2023-11-12T09:26:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:test-16997811824172 2023-11-12T09:26:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-12T09:26:51,218 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997811824172","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781211218"}]},"ts":"1699781211218"} 2023-11-12T09:26:51,220 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997811824172, state=DISABLING in hbase:meta 2023-11-12T09:26:51,222 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set ns3:test-16997811824172 to state=DISABLING 2023-11-12T09:26:51,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16997811824172, region=72ce602541d9ecd5e0168bcd44eb49bd, UNASSIGN}] 2023-11-12T09:26:51,225 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16997811824172, region=72ce602541d9ecd5e0168bcd44eb49bd, UNASSIGN 2023-11-12T09:26:51,226 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=72ce602541d9ecd5e0168bcd44eb49bd, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:51,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; CloseRegionProcedure 72ce602541d9ecd5e0168bcd44eb49bd, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:26:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-12T09:26:51,380 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:51,381 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(119): Close 72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:51,381 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1668): Closing 72ce602541d9ecd5e0168bcd44eb49bd, disabling compactions & flushes 2023-11-12T09:26:51,381 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1690): Closing region ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. 2023-11-12T09:26:51,381 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. 2023-11-12T09:26:51,381 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. after waiting 0 ms 2023-11-12T09:26:51,381 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. 2023-11-12T09:26:51,386 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/72ce602541d9ecd5e0168bcd44eb49bd/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2023-11-12T09:26:51,387 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:26:51,388 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1909): Closed ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd. 2023-11-12T09:26:51,388 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1622): Region close journal for 72ce602541d9ecd5e0168bcd44eb49bd: 2023-11-12T09:26:51,390 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(163): Closed 72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:51,391 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=72ce602541d9ecd5e0168bcd44eb49bd, regionState=CLOSED 2023-11-12T09:26:51,395 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-11-12T09:26:51,395 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; CloseRegionProcedure 72ce602541d9ecd5e0168bcd44eb49bd, server=jenkins-hbase3.apache.org,36287,1699781172353 in 165 msec 2023-11-12T09:26:51,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-11-12T09:26:51,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16997811824172, region=72ce602541d9ecd5e0168bcd44eb49bd, UNASSIGN in 172 msec 2023-11-12T09:26:51,397 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997811824172","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781211397"}]},"ts":"1699781211397"} 2023-11-12T09:26:51,399 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997811824172, state=DISABLED in hbase:meta 2023-11-12T09:26:51,402 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set ns3:test-16997811824172 to state=DISABLED 2023-11-12T09:26:51,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, state=SUCCESS; DisableTableProcedure table=ns3:test-16997811824172 in 190 msec 2023-11-12T09:26:51,479 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:26:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-12T09:26:51,522 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: DISABLE, Table Name: ns3:test-16997811824172, procId: 44 completed 2023-11-12T09:26:51,522 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$16(847): Started truncating ns3:test-16997811824172 2023-11-12T09:26:51,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$6(2481): Client=jenkins//172.31.12.81 truncate ns3:test-16997811824172 2023-11-12T09:26:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=47, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns3:test-16997811824172 preserveSplits=true) 2023-11-12T09:26:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-12T09:26:51,529 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(87): waiting for 'ns3:test-16997811824172' regions in transition 2023-11-12T09:26:51,532 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:51,535 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/72ce602541d9ecd5e0168bcd44eb49bd/f, FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/72ce602541d9ecd5e0168bcd44eb49bd/recovered.edits] 2023-11-12T09:26:51,543 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/72ce602541d9ecd5e0168bcd44eb49bd/recovered.edits/5.seqid to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/ns3/test-16997811824172/72ce602541d9ecd5e0168bcd44eb49bd/recovered.edits/5.seqid 2023-11-12T09:26:51,543 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/72ce602541d9ecd5e0168bcd44eb49bd 2023-11-12T09:26:51,543 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived ns3:test-16997811824172 regions 2023-11-12T09:26:51,549 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:test-16997811824172 from hbase:meta 2023-11-12T09:26:51,551 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:test-16997811824172' descriptor. 2023-11-12T09:26:51,552 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:test-16997811824172' from region states. 2023-11-12T09:26:51,552 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781211552"}]},"ts":"9223372036854775807"} 2023-11-12T09:26:51,554 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:26:51,554 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 72ce602541d9ecd5e0168bcd44eb49bd, NAME => 'ns3:test-16997811824172,,1699781187374.72ce602541d9ecd5e0168bcd44eb49bd.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:26:51,555 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:test-16997811824172' as deleted. 2023-11-12T09:26:51,555 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16997811824172","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781211555"}]},"ts":"9223372036854775807"} 2023-11-12T09:26:51,557 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:test-16997811824172 state from META 2023-11-12T09:26:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-12T09:26:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-12T09:26:51,970 INFO [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8e08148d1dda03a07c8babda3e33ae17, NAME => 'ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16997811824172', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:51,977 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:51,977 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1668): Closing 8e08148d1dda03a07c8babda3e33ae17, disabling compactions & flushes 2023-11-12T09:26:51,977 INFO [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:26:51,978 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:26:51,978 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. after waiting 0 ms 2023-11-12T09:26:51,978 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:26:51,978 INFO [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:26:51,978 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8e08148d1dda03a07c8babda3e33ae17: 2023-11-12T09:26:51,979 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781211979"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781211979"}]},"ts":"1699781211979"} 2023-11-12T09:26:51,981 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:26:51,982 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997811824172","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781211981"}]},"ts":"1699781211981"} 2023-11-12T09:26:51,983 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997811824172, state=ENABLING in hbase:meta 2023-11-12T09:26:51,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997811824172, region=8e08148d1dda03a07c8babda3e33ae17, ASSIGN}] 2023-11-12T09:26:51,992 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997811824172, region=8e08148d1dda03a07c8babda3e33ae17, ASSIGN 2023-11-12T09:26:51,993 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16997811824172, region=8e08148d1dda03a07c8babda3e33ae17, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:26:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-12T09:26:52,145 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=8e08148d1dda03a07c8babda3e33ae17, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:52,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; OpenRegionProcedure 8e08148d1dda03a07c8babda3e33ae17, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:26:52,300 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:52,304 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(130): Open ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:26:52,304 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7254): Opening region: {ENCODED => 8e08148d1dda03a07c8babda3e33ae17, NAME => 'ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:26:52,304 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:26:52,304 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997811824172 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:26:52,305 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(888): Instantiated ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:26:52,305 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7296): checking encryption for 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:26:52,305 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7299): checking classloading for 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:26:52,306 INFO [StoreOpener-8e08148d1dda03a07c8babda3e33ae17-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:26:52,308 INFO [StoreOpener-8e08148d1dda03a07c8babda3e33ae17-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e08148d1dda03a07c8babda3e33ae17 columnFamilyName f 2023-11-12T09:26:52,308 DEBUG [StoreOpener-8e08148d1dda03a07c8babda3e33ae17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:26:52,308 INFO [StoreOpener-8e08148d1dda03a07c8babda3e33ae17-1 {}] regionserver.HStore(324): Store=8e08148d1dda03a07c8babda3e33ae17/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:26:52,309 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:26:52,309 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:26:52,312 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1076): writing seq id for 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:26:52,314 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/8e08148d1dda03a07c8babda3e33ae17/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:26:52,314 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): Opened 8e08148d1dda03a07c8babda3e33ae17; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10295263360, jitterRate=-0.041178882122039795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:26:52,315 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(992): Region open journal for 8e08148d1dda03a07c8babda3e33ae17: 2023-11-12T09:26:52,316 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17., pid=49, masterSystemTime=1699781212300 2023-11-12T09:26:52,318 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:26:52,318 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(158): Opened ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:26:52,318 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=8e08148d1dda03a07c8babda3e33ae17, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:52,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-11-12T09:26:52,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; OpenRegionProcedure 8e08148d1dda03a07c8babda3e33ae17, server=jenkins-hbase3.apache.org,36287,1699781172353 in 173 msec 2023-11-12T09:26:52,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=48, resume processing ppid=47 2023-11-12T09:26:52,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, ppid=47, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16997811824172, region=8e08148d1dda03a07c8babda3e33ae17, ASSIGN in 332 msec 2023-11-12T09:26:52,324 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997811824172","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781212323"}]},"ts":"1699781212323"} 2023-11-12T09:26:52,325 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997811824172, state=ENABLED in hbase:meta 2023-11-12T09:26:52,329 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'ns3:test-16997811824172' completed 2023-11-12T09:26:52,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, state=SUCCESS; TruncateTableProcedure (table=ns3:test-16997811824172 preserveSplits=true) in 806 msec 2023-11-12T09:26:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-12T09:26:52,632 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: TRUNCATE, Table Name: ns3:test-16997811824172, procId: 47 completed 2023-11-12T09:26:52,636 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:26:52,636 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(233): Backup: backup_1699781191676 hdfs://localhost:34955/backupUT/backup_1699781191676/ns3/test-16997811824172/ 2023-11-12T09:26:52,636 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-12T09:26:52,656 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleOverwrite Thread=600 (was 585) Potentially hanging thread: hconnection-0x12da4fa3-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1514736947_16 at /127.0.0.1:44500 [Receiving block BP-1279779094-172.31.12.81-1699781165767:blk_1073741856_1032] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/cluster_2df797c1-14cb-183c-6424-e9a9271897e8/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12da4fa3-shared-pool-15 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1514736947_16 at /127.0.0.1:44486 [Receiving block BP-1279779094-172.31.12.81-1699781165767:blk_1073741855_1031] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1279779094-172.31.12.81-1699781165767:blk_1073741855_1031, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_272620064_16 at /127.0.0.1:44546 [Waiting for operation #33] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1279779094-172.31.12.81-1699781165767:blk_1073741856_1032, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12da4fa3-shared-pool-14 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12da4fa3-shared-pool-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12da4fa3-shared-pool-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/cluster_2df797c1-14cb-183c-6424-e9a9271897e8/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12da4fa3-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1514736947_16 at /127.0.0.1:51298 [Waiting for operation #5] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x21d4d41-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=704 (was 689) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=247 (was 212) - SystemLoadAverage LEAK? -, ProcessCount=168 (was 168), AvailableMemoryMB=5529 (was 5496) - AvailableMemoryMB LEAK? - 2023-11-12T09:26:52,656 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=600 is superior to 500 2023-11-12T09:26:52,674 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleOverwrite Thread=600, OpenFileDescriptor=704, MaxFileDescriptor=60000, SystemLoadAverage=247, ProcessCount=168, AvailableMemoryMB=5528 2023-11-12T09:26:52,674 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=600 is superior to 500 2023-11-12T09:26:52,675 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(173): test full restore on a single table empty table 2023-11-12T09:26:52,676 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1d3a3066 to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:52,714 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1995d80b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:52,717 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:52,720 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36064, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:52,722 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:26:52,726 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54382, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:26:52,741 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:26:52,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:26:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781212746 (current time:1699781212746). 2023-11-12T09:26:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:26:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:26:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x24cc4ed2 to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47c46b9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:52,765 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36078, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x24cc4ed2 to 127.0.0.1:56042 2023-11-12T09:26:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:26:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:26:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:26:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-11-12T09:26:52,771 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:26:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:26:52,772 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:26:52,775 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:26:52,779 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:26:52,792 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:26:52,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0}] 2023-11-12T09:26:52,795 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:26:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:26:52,945 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:52,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2023-11-12T09:26:52,946 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:26:52,947 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=1.51 KB heapSize=2.81 KB 2023-11-12T09:26:52,968 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/6fb01fef96694886ae3b55c3915531db is 171, key is trslm:hdfs://localhost:34955/backupUT\x00ns2:test-16997811824171/meta:log-roll-map/1699781205540/Put/seqid=0 2023-11-12T09:26:52,980 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=792 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/6fb01fef96694886ae3b55c3915531db 2023-11-12T09:26:52,995 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/d247c51ad9b84f17a7cb32cc384f5c0b is 458, key is session:backup_1699781191676/session:context/1699781205630/Put/seqid=0 2023-11-12T09:26:53,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:26:53,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:26:53,404 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=754 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/d247c51ad9b84f17a7cb32cc384f5c0b 2023-11-12T09:26:53,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/6fb01fef96694886ae3b55c3915531db as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db 2023-11-12T09:26:53,419 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db, entries=6, sequenceid=17, filesize=5.8 K 2023-11-12T09:26:53,420 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/d247c51ad9b84f17a7cb32cc384f5c0b as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b 2023-11-12T09:26:53,427 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b, entries=2, sequenceid=17, filesize=5.4 K 2023-11-12T09:26:53,428 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~1.51 KB/1546, heapSize ~2.78 KB/2848, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 482ms, sequenceid=17, compaction requested=false 2023-11-12T09:26:53,428 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:26:53,428 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. for snapshot_backup_system completed. 2023-11-12T09:26:53,428 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:26:53,428 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:26:53,429 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db] hfiles 2023-11-12T09:26:53,429 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db for snapshot=snapshot_backup_system 2023-11-12T09:26:53,429 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b] hfiles 2023-11-12T09:26:53,429 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 for snapshot=snapshot_backup_system 2023-11-12T09:26:53,429 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b for snapshot=snapshot_backup_system 2023-11-12T09:26:53,838 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:26:53,838 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-11-12T09:26:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=51 2023-11-12T09:26:53,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 606338a328e82acae51355d485ed81f0 2023-11-12T09:26:53,840 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:26:53,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-11-12T09:26:53,843 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:26:53,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 in 1.0490 sec 2023-11-12T09:26:53,843 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:26:53,844 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:26:53,844 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:26:53,845 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:26:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:26:54,261 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:26:54,269 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:26:54,269 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:26:54,271 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:26:54,271 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-11-12T09:26:54,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.5040 sec 2023-11-12T09:26:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:26:54,880 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 50 completed 2023-11-12T09:26:54,880 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(115): Backup backup_1699781212714 started at 1699781214880. 2023-11-12T09:26:54,882 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(119): Backup session backup_1699781212714 has been started. 2023-11-12T09:26:54,887 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:26:54,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:26:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:26:54,888 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:26:54,888 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:26:54,889 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:26:54,889 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:26:54,890 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:26:54,890 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:26:54,893 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,893 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:26:54,893 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:26:54,893 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:26:54,893 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,893 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:26:54,894 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:26:54,894 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:26:54,894 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:26:54,894 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:26:54,894 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:26:54,895 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:26:54,895 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:26:54,895 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:26:54,895 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:26:54,896 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:26:54,896 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:26:54,896 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:26:54,898 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,898 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:54,898 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,898 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:26:54,898 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:26:54,898 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:26:54,899 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:26:54,899 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:26:54,899 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:26:54,899 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:26:54,900 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,900 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:26:54,900 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:26:54,900 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7aea9cca[Count = 0] remaining members to acquire global barrier 2023-11-12T09:26:54,900 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:26:54,900 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:54,902 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:54,902 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:54,902 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:54,902 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:26:54,902 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:26:54,902 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,902 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:26:54,902 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,902 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781196322 highest: 1699781196322 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,902 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781196322) roll requested 2023-11-12T09:26:54,914 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781214903, exclude list is [], retry=0 2023-11-12T09:26:54,919 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:26:54,921 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781196322 with entries=19, filesize=6.26 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781214903 2023-11-12T09:26:54,921 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:26:54,921 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781196322 is not closed yet, will try archiving it next time 2023-11-12T09:26:54,921 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781196346) roll requested 2023-11-12T09:26:54,936 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781214921.meta, exclude list is [], retry=0 2023-11-12T09:26:54,940 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:26:54,942 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781196346.meta with entries=24, filesize=6.26 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781214921.meta 2023-11-12T09:26:54,942 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:26:54,942 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781196346.meta is not closed yet, will try archiving it next time 2023-11-12T09:26:54,944 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 42 2023-11-12T09:26:54,944 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781214903 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,944 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781196346.meta not finished, retry = 0 2023-11-12T09:26:54,956 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:26:54,956 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36287,1699781172353' in zk 2023-11-12T09:26:54,959 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,959 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:26:54,959 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:26:54,959 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:26:54,960 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,960 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:26:54,960 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:26:54,961 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:26:54,961 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:26:54,962 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:26:54,962 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,963 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:26:54,963 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:26:54,963 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,964 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36287,1699781172353': 2023-11-12T09:26:54,964 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:26:54,964 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:26:54,964 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:26:54,964 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:26:54,964 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:26:54,964 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:26:54,968 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:26:54,968 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:26:54,968 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:26:54,968 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:26:54,968 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:26:54,968 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:26:54,968 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:26:54,968 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:26:54,968 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:26:54,968 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:26:54,968 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:26:54,969 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,969 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:26:54,969 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:26:54,969 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:26:54,970 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:26:54,970 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,970 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,970 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:26:54,971 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:26:54,971 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,975 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,975 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:26:54,975 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:26:54,975 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:26:54,975 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:54,975 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:54,975 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:26:54,975 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:26:54,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:26:54,975 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:26:54,975 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:26:54,975 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:26:54,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:26:54,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:26:54,976 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:26:54,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:26:54,976 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-12T09:26:54,976 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:26:55,077 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:26:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:26:55,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:26:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781215081 (current time:1699781215081). 2023-11-12T09:26:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:26:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781215080_default_test-1699781182417 VERSION not specified, setting to 2 2023-11-12T09:26:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6abc7f59 to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:26:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec0512f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:26:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:26:55,093 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36080, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:26:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6abc7f59 to 127.0.0.1:56042 2023-11-12T09:26:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:26:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:26:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:26:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-11-12T09:26:55,098 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:26:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-12T09:26:55,099 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:26:55,102 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781215080_default_test-1699781182417 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:26:55,104 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:26:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-12T09:26:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-12T09:26:55,513 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:26:55,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 5f328dc1320f3e032d0f665daf492b24}] 2023-11-12T09:26:55,515 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:26:55,666 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:26:55,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-11-12T09:26:55,667 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. 2023-11-12T09:26:55,668 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing 5f328dc1320f3e032d0f665daf492b24 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-12T09:26:55,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/.tmp/f/66c7b50c95034c07801ae87cbe3e9f4a is 37, key is row10/f:q1/1699781185078/Put/seqid=0 2023-11-12T09:26:55,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-12T09:26:56,089 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/.tmp/f/66c7b50c95034c07801ae87cbe3e9f4a 2023-11-12T09:26:56,096 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/.tmp/f/66c7b50c95034c07801ae87cbe3e9f4a as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/f/66c7b50c95034c07801ae87cbe3e9f4a 2023-11-12T09:26:56,102 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/f/66c7b50c95034c07801ae87cbe3e9f4a, entries=99, sequenceid=103, filesize=8.2 K 2023-11-12T09:26:56,103 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 5f328dc1320f3e032d0f665daf492b24 in 435ms, sequenceid=103, compaction requested=false 2023-11-12T09:26:56,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for 5f328dc1320f3e032d0f665daf492b24: 2023-11-12T09:26:56,105 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. for snapshot_1699781215080_default_test-1699781182417 completed. 2023-11-12T09:26:56,105 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24.' region-info for snapshot=snapshot_1699781215080_default_test-1699781182417 2023-11-12T09:26:56,105 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:26:56,105 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/f/66c7b50c95034c07801ae87cbe3e9f4a] hfiles 2023-11-12T09:26:56,105 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/f/66c7b50c95034c07801ae87cbe3e9f4a for snapshot=snapshot_1699781215080_default_test-1699781182417 2023-11-12T09:26:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-12T09:26:56,511 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. 2023-11-12T09:26:56,511 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-11-12T09:26:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=53 2023-11-12T09:26:56,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781215080_default_test-1699781182417 on region 5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:26:56,513 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:26:56,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-11-12T09:26:56,516 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:26:56,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure 5f328dc1320f3e032d0f665daf492b24 in 1.0010 sec 2023-11-12T09:26:56,516 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:26:56,517 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:26:56,517 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781215080_default_test-1699781182417 2023-11-12T09:26:56,518 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781215080_default_test-1699781182417 2023-11-12T09:26:56,889 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:26:56,928 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:26:56,935 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:26:56,936 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781215080_default_test-1699781182417 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781215080_default_test-1699781182417 2023-11-12T09:26:56,938 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:26:56,938 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-11-12T09:26:56,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1699781215080_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } in 1.8430 sec 2023-11-12T09:26:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-12T09:26:57,205 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: default:test-1699781182417, procId: 52 completed 2023-11-12T09:26:57,205 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781212714 2023-11-12T09:26:57,205 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:26:57,205 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-12T09:26:57,205 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781212714_test-1699781182417 2023-11-12T09:26:57,205 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781215080_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781212714/default/test-1699781182417/ 2023-11-12T09:26:57,205 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:26:57,230 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:26:57,230 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_1909505986_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781212714/default/test-1699781182417, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781212714/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781215080_default_test-1699781182417 2023-11-12T09:26:57,230 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781215080_default_test-1699781182417. 2023-11-12T09:26:57,241 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781215080_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781212714/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781215080_default_test-1699781182417 2023-11-12T09:26:57,264 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-11-12T09:26:57,264 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-11-12T09:26:57,544 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16997811824172' 2023-11-12T09:26:57,662 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:57,662 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:57,663 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:57,663 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:58,617 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-2789510817798369176.jar 2023-11-12T09:26:58,617 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:58,618 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:58,618 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:58,619 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:58,619 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:58,620 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:58,620 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:58,620 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:58,621 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:26:58,621 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:26:58,622 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:26:58,622 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:26:58,623 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:26:58,623 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:26:58,624 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:26:58,624 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:26:58,625 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:26:58,625 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:26:58,625 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:26:58,626 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:26:58,626 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:26:58,627 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:26:58,627 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:26:58,628 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:26:58,629 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:26:58,629 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:26:58,629 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:26:58,630 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:26:58,659 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:26:58,679 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781215080_default_test-1699781182417' hfile list 2023-11-12T09:26:58,682 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-12T09:26:58,753 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781218719/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:26:58,753 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781218719/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:26:58,806 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:26:58,831 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/f/66c7b50c95034c07801ae87cbe3e9f4a output=hdfs://localhost:34955/backupUT/backup_1699781212714/default/test-1699781182417/archive/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/f/66c7b50c95034c07801ae87cbe3e9f4a 2023-11-12T09:26:58,831 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-11-12T09:26:59,788 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:26:59,789 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:26:59,796 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781215080_default_test-1699781182417 2023-11-12T09:26:59,796 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781215080_default_test-1699781182417 finished. 2023-11-12T09:26:59,796 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): test-1699781182417 2023-11-12T09:26:59,803 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781212714 2023-11-12T09:26:59,803 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:27:00,209 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781212714/.backup.manifest 2023-11-12T09:27:00,209 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(358): Backup backup_1699781212714 finished: type=FULL,tablelist=test-1699781182417,targetRootDir=hdfs://localhost:34955/backupUT,startts=1699781214880,completets=1699781219803,bytescopied=0 2023-11-12T09:27:00,209 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:27:00,209 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781215080_default_test-1699781182417 2023-11-12T09:27:00,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781215080_default_test-1699781182417" 2023-11-12T09:27:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781215080_default_test-1699781182417 2023-11-12T09:27:00,214 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781215080_default_test-1699781182417 for backup backup_1699781212714 succeeded. 2023-11-12T09:27:00,215 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:27:00,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:27:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:27:00,223 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:27:00,224 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:27:00,230 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(376): Backup backup_1699781212714 completed. 2023-11-12T09:27:00,230 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:27:00,230 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1d3a3066 to 127.0.0.1:56042 2023-11-12T09:27:00,230 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:00,246 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(178): backup complete 2023-11-12T09:27:00,249 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781212714/.backup.manifest 2023-11-12T09:27:00,251 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781212714/.backup.manifest 2023-11-12T09:27:00,252 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781212714/.backup.manifest 2023-11-12T09:27:00,254 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781212714/.backup.manifest 2023-11-12T09:27:00,254 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'test-1699781182417' to 'test-1699781182417' from full backup image hdfs://localhost:34955/backupUT/backup_1699781212714/default/test-1699781182417 2023-11-12T09:27:00,264 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(476): Truncating exising target table 'test-1699781182417', preserving region splits 2023-11-12T09:27:00,265 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$18(967): Started disable of test-1699781182417 2023-11-12T09:27:00,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable test-1699781182417 2023-11-12T09:27:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=test-1699781182417 2023-11-12T09:27:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:27:00,269 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781182417","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781220269"}]},"ts":"1699781220269"} 2023-11-12T09:27:00,270 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781182417, state=DISABLING in hbase:meta 2023-11-12T09:27:00,272 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set test-1699781182417 to state=DISABLING 2023-11-12T09:27:00,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1699781182417, region=5f328dc1320f3e032d0f665daf492b24, UNASSIGN}] 2023-11-12T09:27:00,275 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1699781182417, region=5f328dc1320f3e032d0f665daf492b24, UNASSIGN 2023-11-12T09:27:00,276 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=5f328dc1320f3e032d0f665daf492b24, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:00,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; CloseRegionProcedure 5f328dc1320f3e032d0f665daf492b24, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:27:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:27:00,429 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:00,429 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(119): Close 5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:27:00,429 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1668): Closing 5f328dc1320f3e032d0f665daf492b24, disabling compactions & flushes 2023-11-12T09:27:00,429 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1690): Closing region test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. 2023-11-12T09:27:00,429 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. 2023-11-12T09:27:00,429 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1778): Acquired close lock on test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. after waiting 0 ms 2023-11-12T09:27:00,429 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1788): Updates disabled for region test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. 2023-11-12T09:27:00,437 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-11-12T09:27:00,437 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:27:00,438 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1909): Closed test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24. 2023-11-12T09:27:00,438 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1622): Region close journal for 5f328dc1320f3e032d0f665daf492b24: 2023-11-12T09:27:00,441 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(163): Closed 5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:27:00,442 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=5f328dc1320f3e032d0f665daf492b24, regionState=CLOSED 2023-11-12T09:27:00,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=56, resume processing ppid=55 2023-11-12T09:27:00,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, ppid=55, state=SUCCESS; CloseRegionProcedure 5f328dc1320f3e032d0f665daf492b24, server=jenkins-hbase3.apache.org,36287,1699781172353 in 167 msec 2023-11-12T09:27:00,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-11-12T09:27:00,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=test-1699781182417, region=5f328dc1320f3e032d0f665daf492b24, UNASSIGN in 173 msec 2023-11-12T09:27:00,450 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781182417","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781220450"}]},"ts":"1699781220450"} 2023-11-12T09:27:00,452 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781182417, state=DISABLED in hbase:meta 2023-11-12T09:27:00,454 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set test-1699781182417 to state=DISABLED 2023-11-12T09:27:00,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; DisableTableProcedure table=test-1699781182417 in 190 msec 2023-11-12T09:27:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:27:00,572 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: DISABLE, Table Name: default:test-1699781182417, procId: 54 completed 2023-11-12T09:27:00,572 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$16(847): Started truncating test-1699781182417 2023-11-12T09:27:00,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$6(2481): Client=jenkins//172.31.12.81 truncate test-1699781182417 2023-11-12T09:27:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=57, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=test-1699781182417 preserveSplits=true) 2023-11-12T09:27:00,576 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(87): waiting for 'test-1699781182417' regions in transition 2023-11-12T09:27:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-12T09:27:00,579 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:27:00,582 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/f, FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/recovered.edits] 2023-11-12T09:27:00,587 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/f/66c7b50c95034c07801ae87cbe3e9f4a to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/f/66c7b50c95034c07801ae87cbe3e9f4a 2023-11-12T09:27:00,591 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/recovered.edits/106.seqid to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/recovered.edits/106.seqid 2023-11-12T09:27:00,591 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:27:00,592 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived test-1699781182417 regions 2023-11-12T09:27:00,596 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of test-1699781182417 from hbase:meta 2023-11-12T09:27:00,598 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'test-1699781182417' descriptor. 2023-11-12T09:27:00,599 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'test-1699781182417' from region states. 2023-11-12T09:27:00,599 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781220599"}]},"ts":"9223372036854775807"} 2023-11-12T09:27:00,600 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:27:00,600 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 5f328dc1320f3e032d0f665daf492b24, NAME => 'test-1699781182417,,1699781182909.5f328dc1320f3e032d0f665daf492b24.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:27:00,601 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'test-1699781182417' as deleted. 2023-11-12T09:27:00,601 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1699781182417","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781220601"}]},"ts":"9223372036854775807"} 2023-11-12T09:27:00,602 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table test-1699781182417 state from META 2023-11-12T09:27:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-12T09:27:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-12T09:27:01,014 INFO [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 15ea62ac691e09a3256d5ddcb7e3267c, NAME => 'test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1699781182417', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:27:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-12T09:27:01,421 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(888): Instantiated test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:27:01,421 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1668): Closing 15ea62ac691e09a3256d5ddcb7e3267c, disabling compactions & flushes 2023-11-12T09:27:01,421 INFO [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1690): Closing region test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:01,421 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:01,421 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. after waiting 0 ms 2023-11-12T09:27:01,421 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:01,421 INFO [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1909): Closed test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:01,421 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1622): Region close journal for 15ea62ac691e09a3256d5ddcb7e3267c: 2023-11-12T09:27:01,423 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1699781221422"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781221422"}]},"ts":"1699781221422"} 2023-11-12T09:27:01,424 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:27:01,425 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781182417","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781221425"}]},"ts":"1699781221425"} 2023-11-12T09:27:01,427 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781182417, state=ENABLING in hbase:meta 2023-11-12T09:27:01,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699781182417, region=15ea62ac691e09a3256d5ddcb7e3267c, ASSIGN}] 2023-11-12T09:27:01,434 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699781182417, region=15ea62ac691e09a3256d5ddcb7e3267c, ASSIGN 2023-11-12T09:27:01,435 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1699781182417, region=15ea62ac691e09a3256d5ddcb7e3267c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:27:01,586 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=15ea62ac691e09a3256d5ddcb7e3267c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:01,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:27:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-12T09:27:01,740 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:01,744 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(130): Open test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:01,744 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7254): Opening region: {ENCODED => 15ea62ac691e09a3256d5ddcb7e3267c, NAME => 'test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:27:01,745 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:27:01,745 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1699781182417 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:01,745 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(888): Instantiated test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:27:01,745 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7296): checking encryption for 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:01,745 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7299): checking classloading for 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:01,746 INFO [StoreOpener-15ea62ac691e09a3256d5ddcb7e3267c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:01,748 INFO [StoreOpener-15ea62ac691e09a3256d5ddcb7e3267c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15ea62ac691e09a3256d5ddcb7e3267c columnFamilyName f 2023-11-12T09:27:01,748 DEBUG [StoreOpener-15ea62ac691e09a3256d5ddcb7e3267c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:27:01,749 INFO [StoreOpener-15ea62ac691e09a3256d5ddcb7e3267c-1 {}] regionserver.HStore(324): Store=15ea62ac691e09a3256d5ddcb7e3267c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:27:01,749 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:01,750 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:01,752 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1076): writing seq id for 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:01,754 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:27:01,755 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): Opened 15ea62ac691e09a3256d5ddcb7e3267c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11443418400, jitterRate=0.06575138866901398}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:27:01,755 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(992): Region open journal for 15ea62ac691e09a3256d5ddcb7e3267c: 2023-11-12T09:27:01,756 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c., pid=59, masterSystemTime=1699781221740 2023-11-12T09:27:01,758 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:01,758 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(158): Opened test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:01,758 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=15ea62ac691e09a3256d5ddcb7e3267c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:01,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=59, resume processing ppid=58 2023-11-12T09:27:01,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c, server=jenkins-hbase3.apache.org,36287,1699781172353 in 172 msec 2023-11-12T09:27:01,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=58, resume processing ppid=57 2023-11-12T09:27:01,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=test-1699781182417, region=15ea62ac691e09a3256d5ddcb7e3267c, ASSIGN in 329 msec 2023-11-12T09:27:01,763 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781182417","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781221763"}]},"ts":"1699781221763"} 2023-11-12T09:27:01,764 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781182417, state=ENABLED in hbase:meta 2023-11-12T09:27:01,766 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(145): truncate 'test-1699781182417' completed 2023-11-12T09:27:01,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, state=SUCCESS; TruncateTableProcedure (table=test-1699781182417 preserveSplits=true) in 1.1930 sec 2023-11-12T09:27:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-12T09:27:02,683 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: TRUNCATE, Table Name: default:test-1699781182417, procId: 57 completed 2023-11-12T09:27:02,687 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34955/backupUT/backup_1699781212714/default/test-1699781182417/archive/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24 from hbase tables test-1699781182417 to tables test-1699781182417 2023-11-12T09:27:02,687 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1699781182417 into test-1699781182417 2023-11-12T09:27:02,689 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781222687 from hdfs://localhost:34955/backupUT/backup_1699781212714/default/test-1699781182417/archive/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24 2023-11-12T09:27:02,690 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x287a1777 to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:27:02,697 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd006ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:02,698 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:02,700 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:51398, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:02,701 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:27:02,701 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1699781182417 2023-11-12T09:27:02,705 DEBUG [hconnection-0x6fd7c5f2-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:02,707 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39244, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:02,708 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1699781182417: 2023-11-12T09:27:02,709 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-12T09:27:02,709 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/partitions_4df5d6cd-205d-43a4-bcb7-d5f72d2a89f8 2023-11-12T09:27:02,716 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:02,717 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:02,718 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:02,718 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,712 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-6059179932449599240.jar 2023-11-12T09:27:03,712 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,713 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,713 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,714 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,714 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,715 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,715 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,716 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,716 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,717 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:27:03,718 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:27:03,718 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:27:03,719 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:27:03,719 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:27:03,720 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:27:03,720 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:27:03,720 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:27:03,721 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:27:03,721 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:27:03,722 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:27:03,722 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:27:03,723 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,724 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,724 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,724 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,724 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,725 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:03,725 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:03,726 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: test-1699781182417 2023-11-12T09:27:03,726 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:27:03,726 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x287a1777 to 127.0.0.1:56042 2023-11-12T09:27:03,726 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:03,726 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-12T09:27:03,726 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-12T09:27:03,727 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:27:03,734 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:27:03,739 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-12T09:27:03,815 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781223782/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:27:03,815 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781223782/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:27:03,872 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34955/backupUT/backup_1699781212714/default/test-1699781182417/archive/data/default/test-1699781182417/5f328dc1320f3e032d0f665daf492b24/f/66c7b50c95034c07801ae87cbe3e9f4a 2023-11-12T09:27:03,876 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-12T09:27:03,897 DEBUG [pool-237-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0ea7c2bb to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:27:03,902 DEBUG [pool-237-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7243da67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:03,903 DEBUG [hconnection-0x5ffda768-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:03,905 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39254, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:03,909 DEBUG [pool-237-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0ea7c2bb to 127.0.0.1:56042 2023-11-12T09:27:03,909 DEBUG [pool-237-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:03,909 DEBUG [pool-237-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-12T09:27:03,909 DEBUG [pool-237-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:27:03,917 DEBUG [pool-237-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781222687/_temporary/0/_temporary/attempt_local108196860_0005_r_000000_0/f/2dc5ce90518d46248fa1cd3ec38e1636 is 37, key is row10/f:q1/1699781185078/Put/seqid=0 2023-11-12T09:27:03,919 WARN [Thread-952 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36287] Specified favored nodes: [jenkins-hbase3.apache.org:36287] 2023-11-12T09:27:04,871 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781222687 2023-11-12T09:27:04,872 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0234a61b to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:27:04,881 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34838411, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:04,883 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:04,884 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39268, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:04,890 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781222687/_SUCCESS 2023-11-12T09:27:04,895 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:04,896 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:51400, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:04,910 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781222687/f/2dc5ce90518d46248fa1cd3ec38e1636 first=Optional[row0] last=Optional[row98] 2023-11-12T09:27:04,912 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c., hostname=jenkins-hbase3.apache.org,36287,1699781172353, seqNum=2 for row with hfile group [{f,hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781222687/f/2dc5ce90518d46248fa1cd3ec38e1636}] 2023-11-12T09:27:04,917 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781222687/f/2dc5ce90518d46248fa1cd3ec38e1636 for inclusion in 15ea62ac691e09a3256d5ddcb7e3267c/f 2023-11-12T09:27:04,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-12T09:27:04,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-12T09:27:04,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegion(2520): Flush status journal for 15ea62ac691e09a3256d5ddcb7e3267c: 2023-11-12T09:27:04,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781222687/f/2dc5ce90518d46248fa1cd3ec38e1636 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__test-1699781182417__k7goveh9h3b0ueh2qepveciuh780vvk04c4f9783md6795bte3avgtbor3m5rln2/f/2dc5ce90518d46248fa1cd3ec38e1636 2023-11-12T09:27:04,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__test-1699781182417__k7goveh9h3b0ueh2qepveciuh780vvk04c4f9783md6795bte3avgtbor3m5rln2/f/2dc5ce90518d46248fa1cd3ec38e1636 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ 2023-11-12T09:27:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1002c92c to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:27:04,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c0709f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:04,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:04,933 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:51416, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:04,939 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39282, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:04,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(428): write bulk load descriptor to backup test-1699781182417 with 1 entries 2023-11-12T09:27:04,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(1602): writing raw bulk path hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ for test-1699781182417 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(435): written 1 rows for bulk load of test-1699781182417 2023-11-12T09:27:04,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:27:04,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1002c92c to 127.0.0.1:56042 2023-11-12T09:27:04,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:04,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__test-1699781182417__k7goveh9h3b0ueh2qepveciuh780vvk04c4f9783md6795bte3avgtbor3m5rln2/f/2dc5ce90518d46248fa1cd3ec38e1636 into 15ea62ac691e09a3256d5ddcb7e3267c/f as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ - updating store file list. 2023-11-12T09:27:04,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ into 15ea62ac691e09a3256d5ddcb7e3267c/f 2023-11-12T09:27:04,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__test-1699781182417__k7goveh9h3b0ueh2qepveciuh780vvk04c4f9783md6795bte3avgtbor3m5rln2/f/2dc5ce90518d46248fa1cd3ec38e1636 into 15ea62ac691e09a3256d5ddcb7e3267c/f (new location: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_) 2023-11-12T09:27:04,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__test-1699781182417__k7goveh9h3b0ueh2qepveciuh780vvk04c4f9783md6795bte3avgtbor3m5rln2/f/2dc5ce90518d46248fa1cd3ec38e1636 2023-11-12T09:27:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1f5f6585 to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:27:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c54e9b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:04,982 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:51428, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:27:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:04,987 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39292, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-12T09:27:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup test-1699781182417 with 1 entries 2023-11-12T09:27:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(1523): writing done bulk path hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ for test-1699781182417 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:05,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(414): written 1 rows for bulk load of test-1699781182417 2023-11-12T09:27:05,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:27:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1f5f6585 to 127.0.0.1:56042 2023-11-12T09:27:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:05,009 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:27:05,010 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0234a61b to 127.0.0.1:56042 2023-11-12T09:27:05,010 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:05,010 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-12T09:27:05,011 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:27:05,011 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(233): Backup: backup_1699781212714 hdfs://localhost:34955/backupUT/backup_1699781212714/default/test-1699781182417/ 2023-11-12T09:27:05,011 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-12T09:27:05,035 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleOverwrite Thread=605 (was 600) Potentially hanging thread: hconnection-0x12da4fa3-shared-pool-23 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56042@0x0234a61b-SendThread(127.0.0.1:56042) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56042@0x1002c92c-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56042@0x1f5f6585-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_272620064_16 at /127.0.0.1:41250 [Waiting for operation #20] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56042@0x0234a61b java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1598) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1570) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1537) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$538/1710005744.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56042@0x1002c92c java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1517) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1541) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$538/1710005744.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56042@0x1f5f6585-SendThread(127.0.0.1:56042) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: hconnection-0x21d4d41-shared-pool-9 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56042@0x0234a61b-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1514736947_16 at /127.0.0.1:51376 [Receiving block BP-1279779094-172.31.12.81-1699781165767:blk_1073741882_1058] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1279779094-172.31.12.81-1699781165767:blk_1073741882_1058, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56042@0x1f5f6585 java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1598) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1570) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1537) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$538/1710005744.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56042@0x1002c92c-SendThread(127.0.0.1:56042) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1394) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: PacketResponder: BP-1279779094-172.31.12.81-1699781165767:blk_1073741883_1059, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1514736947_16 at /127.0.0.1:51384 [Receiving block BP-1279779094-172.31.12.81-1699781165767:blk_1073741883_1059] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x12da4fa3-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=724 (was 704) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=216 (was 247), ProcessCount=168 (was 168), AvailableMemoryMB=5484 (was 5528) 2023-11-12T09:27:05,035 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-12T09:27:05,050 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleDNECommand Thread=605, OpenFileDescriptor=724, MaxFileDescriptor=60000, SystemLoadAverage=216, ProcessCount=168, AvailableMemoryMB=5483 2023-11-12T09:27:05,050 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-12T09:27:05,051 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(280): test restore fails on a single table that does not exist: command-line 2023-11-12T09:27:05,052 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x069a5604 to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:27:05,057 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce72eac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:05,061 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:05,062 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39298, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:05,064 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:05,065 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:51432, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:05,076 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:27:05,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:27:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781225081 (current time:1699781225081). 2023-11-12T09:27:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:27:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:27:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x339478c3 to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:27:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e573d69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:05,090 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39308, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x339478c3 to 127.0.0.1:56042 2023-11-12T09:27:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:27:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:27:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-11-12T09:27:05,094 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:27:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:27:05,095 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:27:05,097 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:27:05,100 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:27:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:27:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:27:05,505 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:27:05,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0}] 2023-11-12T09:27:05,509 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:05,660 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:05,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2023-11-12T09:27:05,661 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:27:05,662 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-12T09:27:05,666 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/0d301e1aaa1e43b798e7fbad9ead0c81 is 169, key is trslm:hdfs://localhost:34955/backupUT\x00test-1699781182417/meta:log-roll-map/1699781219799/Put/seqid=0 2023-11-12T09:27:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:27:05,939 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:27:06,071 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/0d301e1aaa1e43b798e7fbad9ead0c81 2023-11-12T09:27:06,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/e010cb2a3e0b4113b2209bc702d697bd is 452, key is session:backup_1699781212714/session:context/1699781220223/Put/seqid=0 2023-11-12T09:27:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:27:06,484 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/e010cb2a3e0b4113b2209bc702d697bd 2023-11-12T09:27:06,492 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/0d301e1aaa1e43b798e7fbad9ead0c81 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81 2023-11-12T09:27:06,498 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81, entries=4, sequenceid=28, filesize=5.5 K 2023-11-12T09:27:06,499 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/e010cb2a3e0b4113b2209bc702d697bd as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd 2023-11-12T09:27:06,505 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd, entries=2, sequenceid=28, filesize=5.4 K 2023-11-12T09:27:06,506 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 845ms, sequenceid=28, compaction requested=true 2023-11-12T09:27:06,506 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2520): Flush status journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:27:06,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. for snapshot_backup_system completed. 2023-11-12T09:27:06,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:27:06,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:27:06,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81] hfiles 2023-11-12T09:27:06,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db for snapshot=snapshot_backup_system 2023-11-12T09:27:06,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81 for snapshot=snapshot_backup_system 2023-11-12T09:27:06,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd] hfiles 2023-11-12T09:27:06,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 for snapshot=snapshot_backup_system 2023-11-12T09:27:06,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b for snapshot=snapshot_backup_system 2023-11-12T09:27:06,507 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd for snapshot=snapshot_backup_system 2023-11-12T09:27:06,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:27:06,913 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=61 2023-11-12T09:27:06,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=61 2023-11-12T09:27:06,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:06,916 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:06,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-11-12T09:27:06,918 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:27:06,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 in 1.4100 sec 2023-11-12T09:27:06,918 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:27:06,919 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:27:06,919 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:27:06,920 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:27:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:27:07,330 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:27:07,338 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:27:07,339 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:27:07,340 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:27:07,340 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-11-12T09:27:07,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2490 sec 2023-11-12T09:27:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:27:09,202 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 60 completed 2023-11-12T09:27:09,202 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(115): Backup backup_1699781225057 started at 1699781229202. 2023-11-12T09:27:09,204 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(119): Backup session backup_1699781225057 has been started. 2023-11-12T09:27:09,209 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:27:09,210 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:27:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-11-12T09:27:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:27:09,210 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:27:09,210 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:27:09,211 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:27:09,211 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:27:09,212 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:09,212 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:09,213 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:27:09,213 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,213 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:27:09,213 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:27:09,214 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,214 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:27:09,214 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:09,214 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:09,214 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:27:09,215 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:09,215 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:27:09,215 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:27:09,215 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-11-12T09:27:09,215 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:27:09,215 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:27:09,215 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:27:09,216 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:27:09,216 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:27:09,216 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:27:09,218 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,218 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:09,218 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,218 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:09,218 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:09,219 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:27:09,219 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:27:09,219 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:09,219 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:09,219 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:09,220 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,220 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:09,220 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:27:09,220 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@318dbe2[Count = 0] remaining members to acquire global barrier 2023-11-12T09:27:09,220 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:27:09,220 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:09,222 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:09,222 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:09,222 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:09,222 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:27:09,222 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:27:09,222 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,222 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:27:09,222 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,222 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781214903 highest: 1699781214903 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,222 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781214903) roll requested 2023-11-12T09:27:09,234 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781229222, exclude list is [], retry=0 2023-11-12T09:27:09,237 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:27:09,239 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781214903 with entries=17, filesize=5.25 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781229222 2023-11-12T09:27:09,239 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:27:09,239 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781214903 is not closed yet, will try archiving it next time 2023-11-12T09:27:09,239 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781214921) roll requested 2023-11-12T09:27:09,242 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781214903 not finished, retry = 0 2023-11-12T09:27:09,251 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781229239.meta, exclude list is [], retry=0 2023-11-12T09:27:09,254 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:27:09,256 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781214921.meta with entries=12, filesize=3.07 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781229239.meta 2023-11-12T09:27:09,256 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:27:09,256 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781214921.meta is not closed yet, will try archiving it next time 2023-11-12T09:27:09,258 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781214921.meta not finished, retry = 0 2023-11-12T09:27:09,262 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:27:09,262 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781229222 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,271 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:27:09,271 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36287,1699781172353' in zk 2023-11-12T09:27:09,275 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,275 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:27:09,275 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:27:09,275 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:27:09,275 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,276 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:09,276 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:09,276 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:09,276 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:09,277 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:09,277 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,277 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:09,278 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:09,278 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,279 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36287,1699781172353': 2023-11-12T09:27:09,279 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:27:09,279 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:27:09,279 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:27:09,279 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:27:09,279 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:27:09,279 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:27:09,281 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:27:09,281 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:27:09,281 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:27:09,281 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:09,281 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:27:09,281 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:09,281 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:09,281 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:09,281 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:09,281 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:27:09,282 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:27:09,282 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,282 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:09,282 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:09,282 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:09,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:09,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,283 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:09,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:09,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,288 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,288 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:09,288 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:27:09,288 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:09,288 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): regionserver:36287-0x1004cf43a5e0001, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:27:09,288 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,288 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:09,288 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:27:09,288 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:09,288 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:27:09,288 DEBUG [Listener at localhost/39325-EventThread {}] zookeeper.ZKWatcher(604): master:43639-0x1004cf43a5e0000, quorum=127.0.0.1:56042, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:27:09,288 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:27:09,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:27:09,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:27:09,289 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:27:09,289 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:27:09,289 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-12T09:27:09,289 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:27:09,389 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:27:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:27:09,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:27:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781229394 (current time:1699781229394). 2023-11-12T09:27:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:27:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781229393_default_test-1699781182417 VERSION not specified, setting to 2 2023-11-12T09:27:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x60d114ad to 127.0.0.1:56042 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:27:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70133a14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:09,409 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47654, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x60d114ad to 127.0.0.1:56042 2023-11-12T09:27:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:27:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:27:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-11-12T09:27:09,413 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:27:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-12T09:27:09,414 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:27:09,418 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781229393_default_test-1699781182417 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:27:09,421 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:27:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-12T09:27:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-12T09:27:09,828 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:27:09,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c}] 2023-11-12T09:27:09,830 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:09,981 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:09,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-11-12T09:27:09,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:09,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for 15ea62ac691e09a3256d5ddcb7e3267c: 2023-11-12T09:27:09,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. for snapshot_1699781229393_default_test-1699781182417 completed. 2023-11-12T09:27:09,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c.' region-info for snapshot=snapshot_1699781229393_default_test-1699781182417 2023-11-12T09:27:09,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:27:09,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_] hfiles 2023-11-12T09:27:09,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ for snapshot=snapshot_1699781229393_default_test-1699781182417 2023-11-12T09:27:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-12T09:27:10,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:10,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-11-12T09:27:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=63 2023-11-12T09:27:10,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781229393_default_test-1699781182417 on region 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:10,391 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:10,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-11-12T09:27:10,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c in 563 msec 2023-11-12T09:27:10,393 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:27:10,393 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:27:10,394 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:27:10,394 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781229393_default_test-1699781182417 2023-11-12T09:27:10,395 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781229393_default_test-1699781182417 2023-11-12T09:27:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-12T09:27:10,805 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:27:10,811 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:27:10,811 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781229393_default_test-1699781182417 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781229393_default_test-1699781182417 2023-11-12T09:27:10,813 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:27:10,813 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-11-12T09:27:10,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699781229393_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } in 1.4020 sec 2023-11-12T09:27:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-12T09:27:11,520 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: default:test-1699781182417, procId: 62 completed 2023-11-12T09:27:11,520 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781225057 2023-11-12T09:27:11,520 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:27:11,520 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-12T09:27:11,520 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781225057_test-1699781182417 2023-11-12T09:27:11,520 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781229393_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781225057/default/test-1699781182417/ 2023-11-12T09:27:11,521 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:27:11,546 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:27:11,546 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_-2117178617_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781225057/default/test-1699781182417, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781225057/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781229393_default_test-1699781182417 2023-11-12T09:27:11,546 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781229393_default_test-1699781182417. 2023-11-12T09:27:11,555 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781229393_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781225057/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781229393_default_test-1699781182417 2023-11-12T09:27:11,969 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:11,970 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:11,970 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:11,971 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:12,945 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-2643311826526201835.jar 2023-11-12T09:27:12,945 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:12,945 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:12,946 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:12,946 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:12,947 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:12,948 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:12,948 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:12,949 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:12,949 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:12,950 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:27:12,950 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:27:12,951 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:27:12,951 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:27:12,952 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:27:12,952 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:27:12,952 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:27:12,953 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:27:12,953 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:27:12,954 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:27:12,955 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:27:12,955 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:27:12,955 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:12,956 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:12,957 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:12,957 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:12,957 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:12,958 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:12,958 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:12,985 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:27:13,005 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781229393_default_test-1699781182417' hfile list 2023-11-12T09:27:13,007 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-12T09:27:13,079 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781233043/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:27:13,079 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781233043/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:27:13,132 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:27:13,157 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ output=hdfs://localhost:34955/backupUT/backup_1699781225057/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ 2023-11-12T09:27:13,157 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-11-12T09:27:14,115 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:27:14,116 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:27:14,123 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781229393_default_test-1699781182417 2023-11-12T09:27:14,123 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781229393_default_test-1699781182417 finished. 2023-11-12T09:27:14,123 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): test-1699781182417 2023-11-12T09:27:14,133 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781225057 2023-11-12T09:27:14,133 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:27:14,139 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781225057/.backup.manifest 2023-11-12T09:27:14,139 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(358): Backup backup_1699781225057 finished: type=FULL,tablelist=test-1699781182417,targetRootDir=hdfs://localhost:34955/backupUT,startts=1699781229202,completets=1699781234133,bytescopied=0 2023-11-12T09:27:14,140 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:27:14,140 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781229393_default_test-1699781182417 2023-11-12T09:27:14,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781229393_default_test-1699781182417" 2023-11-12T09:27:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781229393_default_test-1699781182417 2023-11-12T09:27:14,144 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781229393_default_test-1699781182417 for backup backup_1699781225057 succeeded. 2023-11-12T09:27:14,145 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:27:14,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:27:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:27:14,152 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:27:14,154 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:27:14,159 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(376): Backup backup_1699781225057 completed. 2023-11-12T09:27:14,159 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:27:14,159 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x069a5604 to 127.0.0.1:56042 2023-11-12T09:27:14,159 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:14,172 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(285): backup complete You have to specify either set name or table list to restore Usage: hbase restore [options] backup_path Path to a backup destination root backup_id Backup image ID to restore table(s) Comma-separated list of tables to restore Options: -c Check restore sequence and dependencies only (does not execute the command) -d Enable debug loggings -h,--help Show usage -m A comma separated list of target tables. If specified, each table in must have a mapping -o Overwrite data if any of the restore target tables exists -q Yarn queue name to run backup restore command on -s Backup set to restore, mutually exclusive with -t (table list) -t Table name list, comma-separated. To enable backup, in hbase-site.xml, set: hbase.backup.enable=true hbase.master.logcleaner.plugins=YOUR_PLUGINS,org.apache.hadoop.hbase.backup.master.BackupLogCleaner hbase.procedure.master.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager hbase.procedure.regionserver.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager hbase.coprocessor.region.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.BackupObserver and restart the cluster For more information please see http://hbase.apache.org/book.html#backuprestore 2023-11-12T09:27:14,202 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleDNECommand Thread=597 (was 605), OpenFileDescriptor=714 (was 724), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=183 (was 216), ProcessCount=168 (was 168), AvailableMemoryMB=5448 (was 5483) 2023-11-12T09:27:14,202 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-11-12T09:27:14,217 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreCheckCommand Thread=597, OpenFileDescriptor=714, MaxFileDescriptor=60000, SystemLoadAverage=183, ProcessCount=168, AvailableMemoryMB=5447 2023-11-12T09:27:14,217 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-11-12T09:27:14,218 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(97): test full restore on a single table: command-line, check only 2023-11-12T09:27:14,223 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69481138, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:14,228 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:14,230 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47662, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:14,235 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:14,236 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47334, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:14,247 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:27:14,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:27:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781234252 (current time:1699781234252). 2023-11-12T09:27:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:27:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:27:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@561b8499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:14,261 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47668, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:27:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:27:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-11-12T09:27:14,265 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:27:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:27:14,266 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:27:14,268 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:27:14,270 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:27:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:27:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:27:14,676 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:27:14,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0}] 2023-11-12T09:27:14,678 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:14,829 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:14,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-11-12T09:27:14,830 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:27:14,830 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-12T09:27:14,834 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/17f27685c473467aa6033e9335fdd0bc is 169, key is trslm:hdfs://localhost:34955/backupUT\x00test-1699781182417/meta:log-roll-map/1699781234126/Put/seqid=0 2023-11-12T09:27:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:27:15,238 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/17f27685c473467aa6033e9335fdd0bc 2023-11-12T09:27:15,249 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/6e038692cf954f1084e0b11d4b62020c is 452, key is session:backup_1699781225057/session:context/1699781234153/Put/seqid=0 2023-11-12T09:27:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:27:15,653 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/6e038692cf954f1084e0b11d4b62020c 2023-11-12T09:27:15,660 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/17f27685c473467aa6033e9335fdd0bc as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc 2023-11-12T09:27:15,665 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc, entries=4, sequenceid=39, filesize=5.5 K 2023-11-12T09:27:15,666 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/6e038692cf954f1084e0b11d4b62020c as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c 2023-11-12T09:27:15,671 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c, entries=2, sequenceid=39, filesize=5.4 K 2023-11-12T09:27:15,672 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 842ms, sequenceid=39, compaction requested=true 2023-11-12T09:27:15,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:27:15,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. for snapshot_backup_system completed. 2023-11-12T09:27:15,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:27:15,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:27:15,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc] hfiles 2023-11-12T09:27:15,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db for snapshot=snapshot_backup_system 2023-11-12T09:27:15,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81 for snapshot=snapshot_backup_system 2023-11-12T09:27:15,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc for snapshot=snapshot_backup_system 2023-11-12T09:27:15,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c] hfiles 2023-11-12T09:27:15,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 for snapshot=snapshot_backup_system 2023-11-12T09:27:15,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b for snapshot=snapshot_backup_system 2023-11-12T09:27:15,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd for snapshot=snapshot_backup_system 2023-11-12T09:27:15,672 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c for snapshot=snapshot_backup_system 2023-11-12T09:27:16,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:27:16,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-11-12T09:27:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=65 2023-11-12T09:27:16,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:16,080 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:16,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-11-12T09:27:16,083 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:27:16,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 in 1.4050 sec 2023-11-12T09:27:16,083 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:27:16,084 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:27:16,084 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:27:16,086 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:27:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:27:16,498 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:27:16,508 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:27:16,509 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:27:16,510 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:27:16,510 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-11-12T09:27:16,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2480 sec 2023-11-12T09:27:16,581 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 79ca523521e828a10ce7dfa8176221f4 changed from -1.0 to 0.0, refreshing cache 2023-11-12T09:27:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:27:18,373 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(115): Backup backup_1699781234223 started at 1699781238373. 2023-11-12T09:27:18,374 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(119): Backup session backup_1699781234223 has been started. 2023-11-12T09:27:18,379 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:27:18,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:27:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:27:18,380 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:27:18,380 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:27:18,381 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:27:18,381 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:27:18,382 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:18,383 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:18,383 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:27:18,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:27:18,384 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:27:18,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:18,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:27:18,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:18,385 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:27:18,385 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:27:18,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:27:18,385 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:27:18,385 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:27:18,386 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:27:18,386 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:27:18,386 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:27:18,388 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:18,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:18,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:18,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:18,389 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:27:18,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:18,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:18,390 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:18,390 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:18,390 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:18,391 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:27:18,391 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@11a85f4e[Count = 0] remaining members to acquire global barrier 2023-11-12T09:27:18,391 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:27:18,391 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:18,393 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:18,393 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:18,393 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:27:18,394 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:27:18,394 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:27:18,394 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:18,394 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781229222 highest: 1699781229222 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:18,394 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781229222) roll requested 2023-11-12T09:27:18,405 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781238394, exclude list is [], retry=0 2023-11-12T09:27:18,409 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:27:18,411 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781229222 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781238394 2023-11-12T09:27:18,411 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:27:18,411 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781229222 is not closed yet, will try archiving it next time 2023-11-12T09:27:18,411 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781229239) roll requested 2023-11-12T09:27:18,423 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781238411.meta, exclude list is [], retry=0 2023-11-12T09:27:18,427 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:27:18,428 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781229239.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781238411.meta 2023-11-12T09:27:18,428 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:27:18,428 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781229239.meta is not closed yet, will try archiving it next time 2023-11-12T09:27:18,431 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781229239.meta to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781229239.meta 2023-11-12T09:27:18,434 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:27:18,434 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781238394 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:18,444 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:27:18,444 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36287,1699781172353' in zk 2023-11-12T09:27:18,449 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:27:18,450 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:27:18,450 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:18,450 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:27:18,450 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:18,450 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:18,451 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:18,451 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:18,451 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:18,452 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:18,452 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:18,452 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:18,453 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:18,453 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36287,1699781172353': 2023-11-12T09:27:18,453 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:27:18,453 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:27:18,453 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:27:18,453 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:27:18,453 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:27:18,453 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:27:18,460 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:27:18,460 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:18,460 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:18,460 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:18,460 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:18,460 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:18,460 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:27:18,460 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:27:18,461 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:18,461 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:18,461 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:18,461 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:18,462 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:18,462 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:18,462 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:18,463 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:18,472 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:27:18,472 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:27:18,472 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:27:18,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:27:18,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:27:18,473 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:27:18,473 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:27:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:27:18,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:27:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781238577 (current time:1699781238577). 2023-11-12T09:27:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:27:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781238577_default_test-1699781182417 VERSION not specified, setting to 2 2023-11-12T09:27:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f69de5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:18,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:18,588 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42352, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:27:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:27:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-11-12T09:27:18,592 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:27:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-12T09:27:18,593 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:27:18,596 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781238577_default_test-1699781182417 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:27:18,598 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:27:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-12T09:27:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-12T09:27:19,003 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:27:19,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c}] 2023-11-12T09:27:19,005 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:19,156 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:19,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=67 2023-11-12T09:27:19,157 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:19,157 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2520): Flush status journal for 15ea62ac691e09a3256d5ddcb7e3267c: 2023-11-12T09:27:19,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. for snapshot_1699781238577_default_test-1699781182417 completed. 2023-11-12T09:27:19,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(240): Storing 'test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c.' region-info for snapshot=snapshot_1699781238577_default_test-1699781182417 2023-11-12T09:27:19,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:27:19,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_] hfiles 2023-11-12T09:27:19,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ for snapshot=snapshot_1699781238577_default_test-1699781182417 2023-11-12T09:27:19,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:19,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=67 2023-11-12T09:27:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=67 2023-11-12T09:27:19,164 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781238577_default_test-1699781182417 on region 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:19,166 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:19,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-11-12T09:27:19,168 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:27:19,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c in 162 msec 2023-11-12T09:27:19,169 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:27:19,169 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:27:19,169 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781238577_default_test-1699781182417 2023-11-12T09:27:19,170 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781238577_default_test-1699781182417 2023-11-12T09:27:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-12T09:27:19,579 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:27:19,586 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:27:19,586 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781238577_default_test-1699781182417 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781238577_default_test-1699781182417 2023-11-12T09:27:19,587 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:27:19,587 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-11-12T09:27:19,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1699781238577_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } in 998 msec 2023-11-12T09:27:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-12T09:27:19,698 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781234223 2023-11-12T09:27:19,698 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:27:19,698 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-12T09:27:19,698 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781234223_test-1699781182417 2023-11-12T09:27:19,698 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781238577_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781234223/default/test-1699781182417/ 2023-11-12T09:27:19,698 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:27:19,722 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:27:19,722 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_-1395843580_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781234223/default/test-1699781182417, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781234223/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781238577_default_test-1699781182417 2023-11-12T09:27:19,722 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781238577_default_test-1699781182417. 2023-11-12T09:27:19,730 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781238577_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781234223/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781238577_default_test-1699781182417 2023-11-12T09:27:20,157 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:20,158 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:20,158 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:20,159 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:21,153 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-6539016466882277868.jar 2023-11-12T09:27:21,154 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:21,154 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:21,154 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:21,155 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:21,155 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:21,156 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:21,156 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:21,156 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:21,157 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:21,157 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:27:21,158 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:27:21,158 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:27:21,159 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:27:21,159 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:27:21,160 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:27:21,160 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:27:21,160 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:27:21,161 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:27:21,161 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:27:21,162 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:27:21,162 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:27:21,163 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:21,163 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:21,164 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:21,164 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:21,164 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:21,165 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:21,166 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:21,209 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781238577_default_test-1699781182417' hfile list 2023-11-12T09:27:21,211 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-12T09:27:21,285 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781241248/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:27:21,285 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781241248/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:27:21,340 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:27:21,365 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ output=hdfs://localhost:34955/backupUT/backup_1699781234223/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ 2023-11-12T09:27:21,365 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-11-12T09:27:22,321 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:27:22,322 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:27:22,328 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781238577_default_test-1699781182417 2023-11-12T09:27:22,328 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781238577_default_test-1699781182417 finished. 2023-11-12T09:27:22,328 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): test-1699781182417 2023-11-12T09:27:22,335 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781234223 2023-11-12T09:27:22,335 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:27:22,740 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781234223/.backup.manifest 2023-11-12T09:27:22,740 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(358): Backup backup_1699781234223 finished: type=FULL,tablelist=test-1699781182417,targetRootDir=hdfs://localhost:34955/backupUT,startts=1699781238373,completets=1699781242335,bytescopied=0 2023-11-12T09:27:22,740 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:27:22,740 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781238577_default_test-1699781182417 2023-11-12T09:27:22,741 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781238577_default_test-1699781182417" 2023-11-12T09:27:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781238577_default_test-1699781182417 2023-11-12T09:27:22,746 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781238577_default_test-1699781182417 for backup backup_1699781234223 succeeded. 2023-11-12T09:27:22,746 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:27:22,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:27:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:27:22,753 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:27:22,755 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:27:22,761 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(376): Backup backup_1699781234223 completed. 2023-11-12T09:27:22,761 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:22,761 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(101): backup complete 2023-11-12T09:27:22,774 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] backup.RestoreDriver(108): Found -check option in restore command, will check and verify the dependencies 2023-11-12T09:27:22,781 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@597af8fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:22,782 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781234223/.backup.manifest 2023-11-12T09:27:22,785 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781234223/.backup.manifest 2023-11-12T09:27:22,785 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.BackupUtils(679): Dependent image(s) from old to new: 2023-11-12T09:27:22,785 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.BackupUtils(688): Backup image: backup_1699781234223 for 'test-1699781182417' is available 2023-11-12T09:27:22,785 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupAdminImpl(511): Checking backup images: OK 2023-11-12T09:27:22,785 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:22,803 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreCheckCommand Thread=596 (was 597), OpenFileDescriptor=706 (was 714), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=178 (was 183), ProcessCount=168 (was 168), AvailableMemoryMB=5369 (was 5447) 2023-11-12T09:27:22,804 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=596 is superior to 500 2023-11-12T09:27:22,818 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingle Thread=596, OpenFileDescriptor=706, MaxFileDescriptor=60000, SystemLoadAverage=178, ProcessCount=168, AvailableMemoryMB=5369 2023-11-12T09:27:22,818 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=596 is superior to 500 2023-11-12T09:27:22,818 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(55): test full restore on a single table empty table 2023-11-12T09:27:22,823 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267229fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:22,826 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:22,828 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42368, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:22,829 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:22,831 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:54434, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:22,847 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:27:22,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:27:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781242852 (current time:1699781242852). 2023-11-12T09:27:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:27:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:27:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4796df60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:22,862 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42382, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:27:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=68, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:27:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 68 2023-11-12T09:27:22,866 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:27:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-12T09:27:22,866 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:27:22,869 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:27:22,871 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:27:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-12T09:27:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-12T09:27:23,277 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:27:23,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0}] 2023-11-12T09:27:23,279 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:23,430 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:23,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=69 2023-11-12T09:27:23,431 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:27:23,431 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-12T09:27:23,436 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/977a0036f3404b628b014de901c01bfd is 169, key is trslm:hdfs://localhost:34955/backupUT\x00test-1699781182417/meta:log-roll-map/1699781242331/Put/seqid=0 2023-11-12T09:27:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-12T09:27:23,847 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/977a0036f3404b628b014de901c01bfd 2023-11-12T09:27:23,855 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/adb71489e30e4e53a48f0bbd3bb4f3b5 is 452, key is session:backup_1699781234223/session:context/1699781242754/Put/seqid=0 2023-11-12T09:27:23,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-12T09:27:24,259 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/adb71489e30e4e53a48f0bbd3bb4f3b5 2023-11-12T09:27:24,268 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/977a0036f3404b628b014de901c01bfd as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd 2023-11-12T09:27:24,273 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd, entries=4, sequenceid=50, filesize=5.5 K 2023-11-12T09:27:24,274 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/adb71489e30e4e53a48f0bbd3bb4f3b5 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5 2023-11-12T09:27:24,279 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5, entries=2, sequenceid=50, filesize=5.4 K 2023-11-12T09:27:24,280 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 848ms, sequenceid=50, compaction requested=true 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(2520): Flush status journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. for snapshot_backup_system completed. 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd] hfiles 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db for snapshot=snapshot_backup_system 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81 for snapshot=snapshot_backup_system 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc for snapshot=snapshot_backup_system 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd for snapshot=snapshot_backup_system 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5] hfiles 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 for snapshot=snapshot_backup_system 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b for snapshot=snapshot_backup_system 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd for snapshot=snapshot_backup_system 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c for snapshot=snapshot_backup_system 2023-11-12T09:27:24,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5 for snapshot=snapshot_backup_system 2023-11-12T09:27:24,687 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:27:24,687 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=69 2023-11-12T09:27:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=69 2023-11-12T09:27:24,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:24,688 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:24,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=69, resume processing ppid=68 2023-11-12T09:27:24,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, ppid=68, state=SUCCESS; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 in 1.4120 sec 2023-11-12T09:27:24,691 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:27:24,691 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:27:24,692 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:27:24,692 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:27:24,694 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:27:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-12T09:27:25,107 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:27:25,120 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:27:25,121 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:27:25,122 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:27:25,122 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 68 2023-11-12T09:27:25,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2590 sec 2023-11-12T09:27:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-12T09:27:26,973 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(115): Backup backup_1699781242823 started at 1699781246973. 2023-11-12T09:27:26,975 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(119): Backup session backup_1699781242823 has been started. 2023-11-12T09:27:26,980 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:27:26,981 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:27:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-11-12T09:27:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:27:26,981 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:27:26,981 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:27:26,982 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:27:26,982 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:27:26,983 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:26,984 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:26,984 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:27:26,984 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:27:26,985 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:27:26,985 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:26,985 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:27:26,985 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:26,985 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:27:26,986 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:27:26,986 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-11-12T09:27:26,986 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:27:26,986 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:27:26,986 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:27:26,987 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:27:26,987 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:27:26,987 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:27:26,995 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:26,995 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:26,995 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:26,995 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:26,995 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:27:26,995 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:26,996 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:26,996 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:26,996 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:26,997 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:26,997 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:27:26,997 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@50b2718b[Count = 0] remaining members to acquire global barrier 2023-11-12T09:27:26,997 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:27:26,997 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:27,002 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:27,002 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:27,002 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:27:27,002 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:27:27,002 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:27:27,002 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:27,002 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781238394 highest: 1699781238394 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:27,003 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781238394) roll requested 2023-11-12T09:27:27,020 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781247003, exclude list is [], retry=0 2023-11-12T09:27:27,027 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:27:27,028 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781238394 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781247003 2023-11-12T09:27:27,028 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:27:27,029 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781238394 is not closed yet, will try archiving it next time 2023-11-12T09:27:27,029 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781238411) roll requested 2023-11-12T09:27:27,029 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781229222 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781229222 2023-11-12T09:27:27,041 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781247029.meta, exclude list is [], retry=0 2023-11-12T09:27:27,044 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:27:27,044 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 42 2023-11-12T09:27:27,044 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781247003 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:27,045 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781238411.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781247029.meta 2023-11-12T09:27:27,045 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:27:27,045 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781238411.meta is not closed yet, will try archiving it next time 2023-11-12T09:27:27,048 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781238411.meta to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781238411.meta 2023-11-12T09:27:27,053 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:27:27,053 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36287,1699781172353' in zk 2023-11-12T09:27:27,056 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:27:27,056 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:27:27,056 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:27:27,056 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:27,057 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:27,057 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:27,058 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:27,058 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:27,058 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:27,059 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:27,059 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:27,059 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:27,059 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:27,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36287,1699781172353': 2023-11-12T09:27:27,060 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:27:27,060 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:27:27,060 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:27:27,060 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:27:27,060 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:27:27,060 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:27:27,062 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:27:27,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:27,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:27,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:27,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:27,062 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:27:27,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:27:27,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:27,063 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:27,063 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:27,063 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:27,063 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:27,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:27,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:27,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:27,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:27,072 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:27:27,072 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:27:27,072 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:27:27,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:27:27,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:27:27,073 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:27:27,073 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:27:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:27:27,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:27:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781247177 (current time:1699781247177). 2023-11-12T09:27:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:27:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781247177_default_test-1699781182417 VERSION not specified, setting to 2 2023-11-12T09:27:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa669ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:27,188 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:56874, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:27:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=70, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:27:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 70 2023-11-12T09:27:27,192 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:27:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-12T09:27:27,193 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:27:27,195 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781247177_default_test-1699781182417 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:27:27,197 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:27:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-12T09:27:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-12T09:27:27,603 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:27:27,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c}] 2023-11-12T09:27:27,605 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:27,756 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:27,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=71 2023-11-12T09:27:27,757 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:27,758 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.HRegion(2520): Flush status journal for 15ea62ac691e09a3256d5ddcb7e3267c: 2023-11-12T09:27:27,758 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. for snapshot_1699781247177_default_test-1699781182417 completed. 2023-11-12T09:27:27,758 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(240): Storing 'test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c.' region-info for snapshot=snapshot_1699781247177_default_test-1699781182417 2023-11-12T09:27:27,758 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:27:27,758 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_] hfiles 2023-11-12T09:27:27,758 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ for snapshot=snapshot_1699781247177_default_test-1699781182417 2023-11-12T09:27:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-12T09:27:28,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:28,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=71 2023-11-12T09:27:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=71 2023-11-12T09:27:28,164 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781247177_default_test-1699781182417 on region 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:28,166 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:28,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=71, resume processing ppid=70 2023-11-12T09:27:28,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, ppid=70, state=SUCCESS; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c in 562 msec 2023-11-12T09:27:28,168 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:27:28,169 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:27:28,169 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:27:28,169 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781247177_default_test-1699781182417 2023-11-12T09:27:28,170 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781247177_default_test-1699781182417 2023-11-12T09:27:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-12T09:27:28,580 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:27:28,586 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:27:28,587 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781247177_default_test-1699781182417 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781247177_default_test-1699781182417 2023-11-12T09:27:28,588 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:27:28,588 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 70 2023-11-12T09:27:28,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1699781247177_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } in 1.3990 sec 2023-11-12T09:27:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-12T09:27:29,298 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781242823 2023-11-12T09:27:29,298 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:27:29,299 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-12T09:27:29,299 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781242823_test-1699781182417 2023-11-12T09:27:29,299 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781247177_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417/ 2023-11-12T09:27:29,299 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:27:29,323 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:27:29,323 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_8608074_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781247177_default_test-1699781182417 2023-11-12T09:27:29,323 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781247177_default_test-1699781182417. 2023-11-12T09:27:29,338 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781247177_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781247177_default_test-1699781182417 2023-11-12T09:27:29,764 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:29,764 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:29,765 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:29,765 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:30,730 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-2948848302708373692.jar 2023-11-12T09:27:30,731 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:30,731 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:30,732 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:30,732 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:30,732 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:30,733 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:30,733 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:30,734 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:30,734 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:30,735 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:27:30,735 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:27:30,736 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:27:30,736 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:27:30,737 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:27:30,737 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:27:30,737 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:27:30,738 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:27:30,738 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:27:30,739 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:27:30,739 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:27:30,739 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:27:30,740 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:30,741 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:30,741 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:30,742 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:30,742 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:30,742 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:30,743 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:30,790 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781247177_default_test-1699781182417' hfile list 2023-11-12T09:27:30,792 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-12T09:27:30,863 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781250829/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:27:30,863 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781250829/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:27:30,943 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:27:30,971 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ output=hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ 2023-11-12T09:27:30,971 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 4.160M/sec 2023-11-12T09:27:31,921 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:27:31,922 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:27:31,928 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781247177_default_test-1699781182417 2023-11-12T09:27:31,928 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781247177_default_test-1699781182417 finished. 2023-11-12T09:27:31,928 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): test-1699781182417 2023-11-12T09:27:31,935 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781242823 2023-11-12T09:27:31,935 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:27:32,340 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781242823/.backup.manifest 2023-11-12T09:27:32,340 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(358): Backup backup_1699781242823 finished: type=FULL,tablelist=test-1699781182417,targetRootDir=hdfs://localhost:34955/backupUT,startts=1699781246973,completets=1699781251935,bytescopied=0 2023-11-12T09:27:32,340 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:27:32,340 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781247177_default_test-1699781182417 2023-11-12T09:27:32,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781247177_default_test-1699781182417" 2023-11-12T09:27:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781247177_default_test-1699781182417 2023-11-12T09:27:32,345 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781247177_default_test-1699781182417 for backup backup_1699781242823 succeeded. 2023-11-12T09:27:32,346 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:27:32,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:27:32,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:27:32,354 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:27:32,356 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:27:32,361 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(376): Backup backup_1699781242823 completed. 2023-11-12T09:27:32,362 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:32,374 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(61): backup complete 2023-11-12T09:27:32,375 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-11-12T09:27:32,376 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781242823/.backup.manifest 2023-11-12T09:27:32,378 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781242823/.backup.manifest 2023-11-12T09:27:32,379 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781242823/.backup.manifest 2023-11-12T09:27:32,381 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781242823/.backup.manifest 2023-11-12T09:27:32,381 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'test-1699781182417' to 'table1' from full backup image hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417 2023-11-12T09:27:32,388 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(487): Creating target table 'table1' 2023-11-12T09:27:32,388 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:32,389 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f in region [hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c] 2023-11-12T09:27:32,393 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ first=row0 last=row98 2023-11-12T09:27:32,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:27:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-11-12T09:27:32,398 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:27:32,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 72 2023-11-12T09:27:32,398 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:27:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-12T09:27:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-12T09:27:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-12T09:27:32,806 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 853c9e7f8846e219b2253ac051e52e17, NAME => 'table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:27:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-12T09:27:33,212 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:27:33,212 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing 853c9e7f8846e219b2253ac051e52e17, disabling compactions & flushes 2023-11-12T09:27:33,212 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17. 2023-11-12T09:27:33,212 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17. 2023-11-12T09:27:33,212 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17. after waiting 0 ms 2023-11-12T09:27:33,212 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17. 2023-11-12T09:27:33,212 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17. 2023-11-12T09:27:33,212 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for 853c9e7f8846e219b2253ac051e52e17: 2023-11-12T09:27:33,214 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:27:33,214 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1699781253214"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781253214"}]},"ts":"1699781253214"} 2023-11-12T09:27:33,215 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:27:33,216 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:27:33,216 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781253216"}]},"ts":"1699781253216"} 2023-11-12T09:27:33,217 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-11-12T09:27:33,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=853c9e7f8846e219b2253ac051e52e17, ASSIGN}] 2023-11-12T09:27:33,226 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=853c9e7f8846e219b2253ac051e52e17, ASSIGN 2023-11-12T09:27:33,226 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=853c9e7f8846e219b2253ac051e52e17, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:27:33,378 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=853c9e7f8846e219b2253ac051e52e17, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:33,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 853c9e7f8846e219b2253ac051e52e17, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:27:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-12T09:27:33,531 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:33,535 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(130): Open table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17. 2023-11-12T09:27:33,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7254): Opening region: {ENCODED => 853c9e7f8846e219b2253ac051e52e17, NAME => 'table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:27:33,535 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:27:33,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 853c9e7f8846e219b2253ac051e52e17 2023-11-12T09:27:33,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(888): Instantiated table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:27:33,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7296): checking encryption for 853c9e7f8846e219b2253ac051e52e17 2023-11-12T09:27:33,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7299): checking classloading for 853c9e7f8846e219b2253ac051e52e17 2023-11-12T09:27:33,537 INFO [StoreOpener-853c9e7f8846e219b2253ac051e52e17-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 853c9e7f8846e219b2253ac051e52e17 2023-11-12T09:27:33,538 INFO [StoreOpener-853c9e7f8846e219b2253ac051e52e17-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 853c9e7f8846e219b2253ac051e52e17 columnFamilyName f 2023-11-12T09:27:33,538 DEBUG [StoreOpener-853c9e7f8846e219b2253ac051e52e17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:27:33,538 INFO [StoreOpener-853c9e7f8846e219b2253ac051e52e17-1 {}] regionserver.HStore(324): Store=853c9e7f8846e219b2253ac051e52e17/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:27:33,539 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17 2023-11-12T09:27:33,539 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17 2023-11-12T09:27:33,541 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1076): writing seq id for 853c9e7f8846e219b2253ac051e52e17 2023-11-12T09:27:33,544 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:27:33,544 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): Opened 853c9e7f8846e219b2253ac051e52e17; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10848821440, jitterRate=0.010375231504440308}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:27:33,545 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(992): Region open journal for 853c9e7f8846e219b2253ac051e52e17: 2023-11-12T09:27:33,546 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17., pid=74, masterSystemTime=1699781253531 2023-11-12T09:27:33,548 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17. 2023-11-12T09:27:33,548 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(158): Opened table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17. 2023-11-12T09:27:33,548 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=853c9e7f8846e219b2253ac051e52e17, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:33,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=74, resume processing ppid=73 2023-11-12T09:27:33,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 853c9e7f8846e219b2253ac051e52e17, server=jenkins-hbase3.apache.org,36287,1699781172353 in 169 msec 2023-11-12T09:27:33,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=73, resume processing ppid=72 2023-11-12T09:27:33,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=table1, region=853c9e7f8846e219b2253ac051e52e17, ASSIGN in 328 msec 2023-11-12T09:27:33,553 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:27:33,553 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781253553"}]},"ts":"1699781253553"} 2023-11-12T09:27:33,555 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-11-12T09:27:33,557 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:27:33,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, state=SUCCESS; CreateTableProcedure table=table1 in 1.1630 sec 2023-11-12T09:27:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-12T09:27:34,508 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c from hbase tables test-1699781182417 to tables table1 2023-11-12T09:27:34,508 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1699781182417 into table1 2023-11-12T09:27:34,510 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781254508 from hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:34,517 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7726d893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:34,518 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:34,519 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45624, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:34,520 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:27:34,520 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-11-12T09:27:34,523 DEBUG [hconnection-0x59765122-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:34,525 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:56888, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:34,527 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-11-12T09:27:34,527 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-12T09:27:34,527 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/partitions_5d16410f-4cfa-4a4d-9347-13c4c5186c0f 2023-11-12T09:27:34,934 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:34,934 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:34,935 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:34,935 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,899 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-6210722892826035180.jar 2023-11-12T09:27:35,899 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,900 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,900 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,901 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,901 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,901 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,902 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,902 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,902 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,903 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:27:35,904 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:27:35,904 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:27:35,904 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:27:35,905 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:27:35,905 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:27:35,906 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:27:35,906 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:27:35,906 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:27:35,907 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:27:35,907 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:27:35,908 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:27:35,908 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,909 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,909 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,909 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,910 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,910 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:35,911 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:35,911 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-11-12T09:27:35,911 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:35,911 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-12T09:27:35,911 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-12T09:27:35,912 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:27:35,924 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-12T09:27:36,003 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781255966/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:27:36,003 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781255966/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:27:36,055 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ 2023-11-12T09:27:36,059 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-12T09:27:36,082 DEBUG [pool-300-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31615892, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:36,084 DEBUG [hconnection-0x7445de35-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:36,086 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49920, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:36,088 DEBUG [pool-300-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:36,090 DEBUG [pool-300-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-12T09:27:36,090 DEBUG [pool-300-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:27:36,100 DEBUG [pool-300-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781254508/_temporary/0/_temporary/attempt_local879105083_0009_r_000000_0/f/f00f964dc68242b3ab04eaf90fb4023d is 37, key is row10/f:q1/1699781185078/Put/seqid=0 2023-11-12T09:27:36,102 WARN [Thread-1257 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36287] Specified favored nodes: [jenkins-hbase3.apache.org:36287] 2023-11-12T09:27:37,051 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781254508 2023-11-12T09:27:37,057 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58cb3d6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:37,058 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:37,060 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49930, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:37,065 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781254508/_SUCCESS 2023-11-12T09:27:37,069 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:37,070 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39732, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:37,079 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781254508/f/f00f964dc68242b3ab04eaf90fb4023d first=Optional[row0] last=Optional[row98] 2023-11-12T09:27:37,080 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17., hostname=jenkins-hbase3.apache.org,36287,1699781172353, seqNum=2 for row with hfile group [{f,hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781254508/f/f00f964dc68242b3ab04eaf90fb4023d}] 2023-11-12T09:27:37,085 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781254508/f/f00f964dc68242b3ab04eaf90fb4023d for inclusion in 853c9e7f8846e219b2253ac051e52e17/f 2023-11-12T09:27:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-12T09:27:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-12T09:27:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegion(2520): Flush status journal for 853c9e7f8846e219b2253ac051e52e17: 2023-11-12T09:27:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781254508/f/f00f964dc68242b3ab04eaf90fb4023d to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__table1__2nrk9u6jb71qbduq6fa2mjsmvust61fapq186nle3prmcjenbm5f6juce212neea/f/f00f964dc68242b3ab04eaf90fb4023d 2023-11-12T09:27:37,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__table1__2nrk9u6jb71qbduq6fa2mjsmvust61fapq186nle3prmcjenbm5f6juce212neea/f/f00f964dc68242b3ab04eaf90fb4023d as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17/f/74a5af8c5d424fc9b252884bccac3e6e_SeqId_4_ 2023-11-12T09:27:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a81bf9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:37,100 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39738, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:37,105 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49944, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:37,121 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__table1__2nrk9u6jb71qbduq6fa2mjsmvust61fapq186nle3prmcjenbm5f6juce212neea/f/f00f964dc68242b3ab04eaf90fb4023d into 853c9e7f8846e219b2253ac051e52e17/f as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17/f/74a5af8c5d424fc9b252884bccac3e6e_SeqId_4_ - updating store file list. 2023-11-12T09:27:37,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17/f/74a5af8c5d424fc9b252884bccac3e6e_SeqId_4_ into 853c9e7f8846e219b2253ac051e52e17/f 2023-11-12T09:27:37,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__table1__2nrk9u6jb71qbduq6fa2mjsmvust61fapq186nle3prmcjenbm5f6juce212neea/f/f00f964dc68242b3ab04eaf90fb4023d into 853c9e7f8846e219b2253ac051e52e17/f (new location: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17/f/74a5af8c5d424fc9b252884bccac3e6e_SeqId_4_) 2023-11-12T09:27:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__table1__2nrk9u6jb71qbduq6fa2mjsmvust61fapq186nle3prmcjenbm5f6juce212neea/f/f00f964dc68242b3ab04eaf90fb4023d 2023-11-12T09:27:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d54f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:37,138 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39750, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:27:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:37,142 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49950, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-12T09:27:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:37,157 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:37,157 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-12T09:27:37,157 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:27:37,158 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(233): Backup: backup_1699781242823 hdfs://localhost:34955/backupUT/backup_1699781242823/default/test-1699781182417/ 2023-11-12T09:27:37,158 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-12T09:27:37,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable table1 2023-11-12T09:27:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=75, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-11-12T09:27:37,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2023-11-12T09:27:37,162 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781257162"}]},"ts":"1699781257162"} 2023-11-12T09:27:37,163 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-11-12T09:27:37,165 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-11-12T09:27:37,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=853c9e7f8846e219b2253ac051e52e17, UNASSIGN}] 2023-11-12T09:27:37,167 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=853c9e7f8846e219b2253ac051e52e17, UNASSIGN 2023-11-12T09:27:37,167 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=853c9e7f8846e219b2253ac051e52e17, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:37,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; CloseRegionProcedure 853c9e7f8846e219b2253ac051e52e17, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:27:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2023-11-12T09:27:37,320 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:37,320 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(119): Close 853c9e7f8846e219b2253ac051e52e17 2023-11-12T09:27:37,320 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1668): Closing 853c9e7f8846e219b2253ac051e52e17, disabling compactions & flushes 2023-11-12T09:27:37,320 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1690): Closing region table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17. 2023-11-12T09:27:37,320 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17. 2023-11-12T09:27:37,320 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1778): Acquired close lock on table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17. after waiting 0 ms 2023-11-12T09:27:37,320 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1788): Updates disabled for region table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17. 2023-11-12T09:27:37,326 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-12T09:27:37,327 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:27:37,327 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1909): Closed table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17. 2023-11-12T09:27:37,327 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1622): Region close journal for 853c9e7f8846e219b2253ac051e52e17: 2023-11-12T09:27:37,328 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(163): Closed 853c9e7f8846e219b2253ac051e52e17 2023-11-12T09:27:37,329 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=853c9e7f8846e219b2253ac051e52e17, regionState=CLOSED 2023-11-12T09:27:37,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=77, resume processing ppid=76 2023-11-12T09:27:37,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=77, ppid=76, state=SUCCESS; CloseRegionProcedure 853c9e7f8846e219b2253ac051e52e17, server=jenkins-hbase3.apache.org,36287,1699781172353 in 162 msec 2023-11-12T09:27:37,333 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=76, resume processing ppid=75 2023-11-12T09:27:37,333 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=table1, region=853c9e7f8846e219b2253ac051e52e17, UNASSIGN in 165 msec 2023-11-12T09:27:37,333 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781257333"}]},"ts":"1699781257333"} 2023-11-12T09:27:37,335 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-11-12T09:27:37,338 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-11-12T09:27:37,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, state=SUCCESS; DisableTableProcedure table=table1 in 179 msec 2023-11-12T09:27:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2023-11-12T09:27:37,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$5(2449): Client=jenkins//172.31.12.81 delete table1 2023-11-12T09:27:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-11-12T09:27:37,480 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:27:37,481 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:27:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=78 2023-11-12T09:27:37,483 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17 2023-11-12T09:27:37,485 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17/f, FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17/recovered.edits] 2023-11-12T09:27:37,489 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17/f/74a5af8c5d424fc9b252884bccac3e6e_SeqId_4_ to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/default/table1/853c9e7f8846e219b2253ac051e52e17/f/74a5af8c5d424fc9b252884bccac3e6e_SeqId_4_ 2023-11-12T09:27:37,492 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17/recovered.edits/6.seqid to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/default/table1/853c9e7f8846e219b2253ac051e52e17/recovered.edits/6.seqid 2023-11-12T09:27:37,493 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/853c9e7f8846e219b2253ac051e52e17 2023-11-12T09:27:37,493 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-11-12T09:27:37,495 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:27:37,497 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-11-12T09:27:37,499 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-11-12T09:27:37,500 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:27:37,500 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-11-12T09:27:37,500 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781257500"}]},"ts":"9223372036854775807"} 2023-11-12T09:27:37,501 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:27:37,501 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 853c9e7f8846e219b2253ac051e52e17, NAME => 'table1,,1699781252394.853c9e7f8846e219b2253ac051e52e17.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:27:37,501 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-11-12T09:27:37,502 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781257501"}]},"ts":"9223372036854775807"} 2023-11-12T09:27:37,503 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-11-12T09:27:37,505 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:27:37,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=78, state=SUCCESS; DeleteTableProcedure table=table1 in 33 msec 2023-11-12T09:27:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=78 2023-11-12T09:27:37,600 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingle Thread=595 (was 596), OpenFileDescriptor=702 (was 706), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=146 (was 178), ProcessCount=168 (was 168), AvailableMemoryMB=5308 (was 5369) 2023-11-12T09:27:37,600 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=595 is superior to 500 2023-11-12T09:27:37,616 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleOverwriteCommand Thread=595, OpenFileDescriptor=702, MaxFileDescriptor=60000, SystemLoadAverage=146, ProcessCount=168, AvailableMemoryMB=5307 2023-11-12T09:27:37,616 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=595 is superior to 500 2023-11-12T09:27:37,616 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(192): test full restore on a single table empty table: command-line 2023-11-12T09:27:37,621 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ae4f33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:37,624 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:37,626 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49960, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:37,627 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:37,629 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39752, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:37,639 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:27:37,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:27:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781257644 (current time:1699781257644). 2023-11-12T09:27:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:27:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:27:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f0b92a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:37,653 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49966, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:27:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=79, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:27:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-11-12T09:27:37,657 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:27:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-12T09:27:37,657 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:27:37,660 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:27:37,662 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:27:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-12T09:27:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-12T09:27:38,068 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:27:38,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0}] 2023-11-12T09:27:38,070 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:38,220 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:38,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=80 2023-11-12T09:27:38,221 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:27:38,222 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-12T09:27:38,225 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/3b495e5a835a4c48b378e5ad24d02c00 is 169, key is trslm:hdfs://localhost:34955/backupUT\x00test-1699781182417/meta:log-roll-map/1699781251931/Put/seqid=0 2023-11-12T09:27:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-12T09:27:38,630 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/3b495e5a835a4c48b378e5ad24d02c00 2023-11-12T09:27:38,638 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/40236c1a3b484da8a8d88269f03914e8 is 452, key is session:backup_1699781242823/session:context/1699781252355/Put/seqid=0 2023-11-12T09:27:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-12T09:27:39,043 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/40236c1a3b484da8a8d88269f03914e8 2023-11-12T09:27:39,049 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/3b495e5a835a4c48b378e5ad24d02c00 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00 2023-11-12T09:27:39,054 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00, entries=4, sequenceid=61, filesize=5.5 K 2023-11-12T09:27:39,055 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/40236c1a3b484da8a8d88269f03914e8 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8 2023-11-12T09:27:39,059 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8, entries=2, sequenceid=61, filesize=5.4 K 2023-11-12T09:27:39,060 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 839ms, sequenceid=61, compaction requested=true 2023-11-12T09:27:39,060 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(2520): Flush status journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:27:39,060 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. for snapshot_backup_system completed. 2023-11-12T09:27:39,060 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:27:39,060 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:27:39,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00] hfiles 2023-11-12T09:27:39,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db for snapshot=snapshot_backup_system 2023-11-12T09:27:39,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81 for snapshot=snapshot_backup_system 2023-11-12T09:27:39,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc for snapshot=snapshot_backup_system 2023-11-12T09:27:39,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd for snapshot=snapshot_backup_system 2023-11-12T09:27:39,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00 for snapshot=snapshot_backup_system 2023-11-12T09:27:39,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8] hfiles 2023-11-12T09:27:39,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 for snapshot=snapshot_backup_system 2023-11-12T09:27:39,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b for snapshot=snapshot_backup_system 2023-11-12T09:27:39,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd for snapshot=snapshot_backup_system 2023-11-12T09:27:39,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c for snapshot=snapshot_backup_system 2023-11-12T09:27:39,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5 for snapshot=snapshot_backup_system 2023-11-12T09:27:39,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8 for snapshot=snapshot_backup_system 2023-11-12T09:27:39,078 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:27:39,466 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:27:39,466 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=80 2023-11-12T09:27:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=80 2023-11-12T09:27:39,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:39,468 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:39,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=80, resume processing ppid=79 2023-11-12T09:27:39,470 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:27:39,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=80, ppid=79, state=SUCCESS; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 in 1.4010 sec 2023-11-12T09:27:39,471 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:27:39,471 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:27:39,471 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:27:39,472 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:27:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-12T09:27:39,885 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:27:39,895 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:27:39,896 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:27:39,897 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:27:39,897 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-11-12T09:27:39,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2430 sec 2023-11-12T09:27:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-12T09:27:41,765 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(115): Backup backup_1699781257621 started at 1699781261764. 2023-11-12T09:27:41,766 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(119): Backup session backup_1699781257621 has been started. 2023-11-12T09:27:41,771 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:27:41,772 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:27:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:27:41,772 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:27:41,772 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:27:41,773 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:27:41,773 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:27:41,774 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:41,775 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:41,775 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:27:41,775 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:27:41,775 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:27:41,776 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:41,776 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:27:41,776 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:41,776 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:27:41,777 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:27:41,777 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:27:41,777 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:27:41,777 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:27:41,777 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:27:41,777 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:27:41,778 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:27:41,780 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:41,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:41,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:41,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:41,780 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:27:41,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:41,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:41,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:41,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:41,782 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:41,782 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:27:41,782 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7e10bb9b[Count = 0] remaining members to acquire global barrier 2023-11-12T09:27:41,782 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:27:41,782 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:41,784 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:41,784 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:41,784 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:27:41,784 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:27:41,784 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:27:41,784 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:41,784 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781247003 highest: 1699781247003 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:41,784 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781247003) roll requested 2023-11-12T09:27:41,795 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781261784, exclude list is [], retry=0 2023-11-12T09:27:41,801 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:27:41,803 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781247003 with entries=13, filesize=3.09 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781261784 2023-11-12T09:27:41,803 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:27:41,803 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781247003 is not closed yet, will try archiving it next time 2023-11-12T09:27:41,803 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781247029) roll requested 2023-11-12T09:27:41,803 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781238394 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781238394 2023-11-12T09:27:41,815 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781261803.meta, exclude list is [], retry=0 2023-11-12T09:27:41,817 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:27:41,819 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781247029.meta with entries=12, filesize=2.73 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781261803.meta 2023-11-12T09:27:41,819 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:27:41,819 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781247029.meta is not closed yet, will try archiving it next time 2023-11-12T09:27:41,824 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:27:41,824 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781261784 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:41,833 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:27:41,833 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36287,1699781172353' in zk 2023-11-12T09:27:41,838 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:27:41,838 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:41,838 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:27:41,838 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:41,838 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:27:41,839 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:41,839 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:41,840 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:41,840 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:41,840 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:41,841 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:41,841 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:41,841 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:41,842 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36287,1699781172353': 2023-11-12T09:27:41,842 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:27:41,842 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:27:41,842 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:27:41,842 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:27:41,842 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:27:41,842 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:27:41,844 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:27:41,844 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:41,844 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:41,844 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:41,844 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:41,845 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:27:41,845 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:27:41,845 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:41,845 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:41,845 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:41,846 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:41,846 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:41,847 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:41,847 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:41,847 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:41,848 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:41,852 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:27:41,852 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:27:41,852 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:27:41,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:27:41,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:27:41,853 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:27:41,853 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:27:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:27:41,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:27:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781261957 (current time:1699781261957). 2023-11-12T09:27:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:27:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781261956_default_test-1699781182417 VERSION not specified, setting to 2 2023-11-12T09:27:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@717df5be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:41,967 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49970, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:27:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=81, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:27:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 81 2023-11-12T09:27:41,971 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:27:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-12T09:27:41,972 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:27:41,974 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781261956_default_test-1699781182417 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:27:41,976 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:27:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-12T09:27:42,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-12T09:27:42,382 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:27:42,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c}] 2023-11-12T09:27:42,384 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:42,535 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:42,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=82 2023-11-12T09:27:42,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:42,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.HRegion(2520): Flush status journal for 15ea62ac691e09a3256d5ddcb7e3267c: 2023-11-12T09:27:42,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. for snapshot_1699781261956_default_test-1699781182417 completed. 2023-11-12T09:27:42,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(240): Storing 'test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c.' region-info for snapshot=snapshot_1699781261956_default_test-1699781182417 2023-11-12T09:27:42,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:27:42,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_] hfiles 2023-11-12T09:27:42,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ for snapshot=snapshot_1699781261956_default_test-1699781182417 2023-11-12T09:27:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-12T09:27:42,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:42,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=82 2023-11-12T09:27:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=82 2023-11-12T09:27:42,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781261956_default_test-1699781182417 on region 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:42,944 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:42,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=82, resume processing ppid=81 2023-11-12T09:27:42,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=82, ppid=81, state=SUCCESS; SnapshotRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c in 563 msec 2023-11-12T09:27:42,946 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:27:42,946 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:27:42,947 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:27:42,947 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781261956_default_test-1699781182417 2023-11-12T09:27:42,948 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781261956_default_test-1699781182417 2023-11-12T09:27:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-12T09:27:43,357 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:27:43,363 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:27:43,364 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781261956_default_test-1699781182417 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781261956_default_test-1699781182417 2023-11-12T09:27:43,365 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:27:43,365 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 81 2023-11-12T09:27:43,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1699781261956_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } in 1.3970 sec 2023-11-12T09:27:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-12T09:27:44,078 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781257621 2023-11-12T09:27:44,078 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:27:44,078 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-12T09:27:44,078 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781257621_test-1699781182417 2023-11-12T09:27:44,078 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781261956_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781257621/default/test-1699781182417/ 2023-11-12T09:27:44,078 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:27:44,101 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:27:44,101 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_1184762392_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781257621/default/test-1699781182417, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781257621/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781261956_default_test-1699781182417 2023-11-12T09:27:44,101 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781261956_default_test-1699781182417. 2023-11-12T09:27:44,109 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781261956_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781257621/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781261956_default_test-1699781182417 2023-11-12T09:27:44,527 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:44,528 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:44,528 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:44,529 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:45,482 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-6414545408311067779.jar 2023-11-12T09:27:45,483 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:45,483 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:45,484 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:45,484 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:45,485 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:45,485 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:45,485 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:45,486 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:45,486 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:45,487 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:27:45,487 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:27:45,488 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:27:45,488 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:27:45,488 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:27:45,489 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:27:45,489 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:27:45,489 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:27:45,490 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:27:45,490 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:27:45,491 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:27:45,491 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:27:45,492 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:45,492 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:45,493 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:45,493 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:45,494 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:27:45,494 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:45,495 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:45,538 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781261956_default_test-1699781182417' hfile list 2023-11-12T09:27:45,540 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-12T09:27:45,609 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781265576/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:27:45,609 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781265576/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:27:45,664 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:27:45,712 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ output=hdfs://localhost:34955/backupUT/backup_1699781257621/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ 2023-11-12T09:27:45,712 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 4.160M/sec 2023-11-12T09:27:46,641 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:27:46,642 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:27:46,649 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781261956_default_test-1699781182417 2023-11-12T09:27:46,649 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781261956_default_test-1699781182417 finished. 2023-11-12T09:27:46,649 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): test-1699781182417 2023-11-12T09:27:46,655 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781257621 2023-11-12T09:27:46,655 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:27:47,061 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781257621/.backup.manifest 2023-11-12T09:27:47,061 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(358): Backup backup_1699781257621 finished: type=FULL,tablelist=test-1699781182417,targetRootDir=hdfs://localhost:34955/backupUT,startts=1699781261764,completets=1699781266655,bytescopied=0 2023-11-12T09:27:47,061 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:27:47,061 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781261956_default_test-1699781182417 2023-11-12T09:27:47,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781261956_default_test-1699781182417" 2023-11-12T09:27:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781261956_default_test-1699781182417 2023-11-12T09:27:47,065 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781261956_default_test-1699781182417 for backup backup_1699781257621 succeeded. 2023-11-12T09:27:47,066 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:27:47,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:27:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:27:47,073 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:27:47,075 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:27:47,080 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(376): Backup backup_1699781257621 completed. 2023-11-12T09:27:47,081 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:47,091 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(196): backup complete 2023-11-12T09:27:47,092 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] backup.RestoreDriver(101): Found -overwrite option in restore command, will overwrite to existing table if any in the restore target 2023-11-12T09:27:47,097 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2b225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:47,098 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:47,102 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43226, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:47,104 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781257621/.backup.manifest 2023-11-12T09:27:47,107 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781257621/.backup.manifest 2023-11-12T09:27:47,108 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781257621/.backup.manifest 2023-11-12T09:27:47,110 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781257621/.backup.manifest 2023-11-12T09:27:47,110 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'test-1699781182417' to 'test-1699781182417' from full backup image hdfs://localhost:34955/backupUT/backup_1699781257621/default/test-1699781182417 2023-11-12T09:27:47,117 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(476): Truncating exising target table 'test-1699781182417', preserving region splits 2023-11-12T09:27:47,117 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:47,119 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38108, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:47,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable test-1699781182417 2023-11-12T09:27:47,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=83, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=test-1699781182417 2023-11-12T09:27:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=83 2023-11-12T09:27:47,123 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781182417","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781267123"}]},"ts":"1699781267123"} 2023-11-12T09:27:47,124 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781182417, state=DISABLING in hbase:meta 2023-11-12T09:27:47,126 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set test-1699781182417 to state=DISABLING 2023-11-12T09:27:47,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1699781182417, region=15ea62ac691e09a3256d5ddcb7e3267c, UNASSIGN}] 2023-11-12T09:27:47,128 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1699781182417, region=15ea62ac691e09a3256d5ddcb7e3267c, UNASSIGN 2023-11-12T09:27:47,129 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=84 updating hbase:meta row=15ea62ac691e09a3256d5ddcb7e3267c, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:47,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE; CloseRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:27:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=83 2023-11-12T09:27:47,281 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:47,281 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(119): Close 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:47,282 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1668): Closing 15ea62ac691e09a3256d5ddcb7e3267c, disabling compactions & flushes 2023-11-12T09:27:47,282 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1690): Closing region test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:47,282 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:47,282 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1778): Acquired close lock on test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. after waiting 0 ms 2023-11-12T09:27:47,282 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1788): Updates disabled for region test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:47,288 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/recovered.edits/10.seqid, newMaxSeqId=10, maxSeqId=1 2023-11-12T09:27:47,288 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:27:47,289 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1909): Closed test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c. 2023-11-12T09:27:47,289 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1622): Region close journal for 15ea62ac691e09a3256d5ddcb7e3267c: 2023-11-12T09:27:47,290 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(163): Closed 15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:47,291 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=84 updating hbase:meta row=15ea62ac691e09a3256d5ddcb7e3267c, regionState=CLOSED 2023-11-12T09:27:47,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=85, resume processing ppid=84 2023-11-12T09:27:47,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=85, ppid=84, state=SUCCESS; CloseRegionProcedure 15ea62ac691e09a3256d5ddcb7e3267c, server=jenkins-hbase3.apache.org,36287,1699781172353 in 162 msec 2023-11-12T09:27:47,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=84, resume processing ppid=83 2023-11-12T09:27:47,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=84, ppid=83, state=SUCCESS; TransitRegionStateProcedure table=test-1699781182417, region=15ea62ac691e09a3256d5ddcb7e3267c, UNASSIGN in 166 msec 2023-11-12T09:27:47,295 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781182417","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781267295"}]},"ts":"1699781267295"} 2023-11-12T09:27:47,296 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781182417, state=DISABLED in hbase:meta 2023-11-12T09:27:47,308 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set test-1699781182417 to state=DISABLED 2023-11-12T09:27:47,309 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=83, state=SUCCESS; DisableTableProcedure table=test-1699781182417 in 188 msec 2023-11-12T09:27:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=83 2023-11-12T09:27:47,426 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$6(2481): Client=jenkins//172.31.12.81 truncate test-1699781182417 2023-11-12T09:27:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=86, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=test-1699781182417 preserveSplits=true) 2023-11-12T09:27:47,428 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(87): waiting for 'test-1699781182417' regions in transition 2023-11-12T09:27:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=86 2023-11-12T09:27:47,431 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:47,433 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f, FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/recovered.edits] 2023-11-12T09:27:47,437 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ 2023-11-12T09:27:47,441 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/recovered.edits/10.seqid to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/recovered.edits/10.seqid 2023-11-12T09:27:47,441 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:47,441 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived test-1699781182417 regions 2023-11-12T09:27:47,446 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of test-1699781182417 from hbase:meta 2023-11-12T09:27:47,447 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'test-1699781182417' descriptor. 2023-11-12T09:27:47,448 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'test-1699781182417' from region states. 2023-11-12T09:27:47,448 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781267448"}]},"ts":"9223372036854775807"} 2023-11-12T09:27:47,450 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:27:47,450 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 15ea62ac691e09a3256d5ddcb7e3267c, NAME => 'test-1699781182417,,1699781220593.15ea62ac691e09a3256d5ddcb7e3267c.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:27:47,450 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'test-1699781182417' as deleted. 2023-11-12T09:27:47,450 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1699781182417","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781267450"}]},"ts":"9223372036854775807"} 2023-11-12T09:27:47,451 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table test-1699781182417 state from META 2023-11-12T09:27:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=86 2023-11-12T09:27:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=86 2023-11-12T09:27:47,861 INFO [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3ca9201e34d84739ee34b92f9445e4a2, NAME => 'test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1699781182417', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:27:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=86 2023-11-12T09:27:48,267 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(888): Instantiated test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:27:48,267 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1668): Closing 3ca9201e34d84739ee34b92f9445e4a2, disabling compactions & flushes 2023-11-12T09:27:48,267 INFO [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1690): Closing region test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:27:48,267 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:27:48,267 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. after waiting 0 ms 2023-11-12T09:27:48,267 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:27:48,267 INFO [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1909): Closed test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:27:48,267 DEBUG [RegionOpenAndInit-test-1699781182417-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3ca9201e34d84739ee34b92f9445e4a2: 2023-11-12T09:27:48,268 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1699781268268"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781268268"}]},"ts":"1699781268268"} 2023-11-12T09:27:48,270 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:27:48,270 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781182417","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781268270"}]},"ts":"1699781268270"} 2023-11-12T09:27:48,272 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781182417, state=ENABLING in hbase:meta 2023-11-12T09:27:48,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699781182417, region=3ca9201e34d84739ee34b92f9445e4a2, ASSIGN}] 2023-11-12T09:27:48,278 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699781182417, region=3ca9201e34d84739ee34b92f9445e4a2, ASSIGN 2023-11-12T09:27:48,278 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1699781182417, region=3ca9201e34d84739ee34b92f9445e4a2, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:27:48,430 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=87 updating hbase:meta row=3ca9201e34d84739ee34b92f9445e4a2, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:48,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; OpenRegionProcedure 3ca9201e34d84739ee34b92f9445e4a2, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:27:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=86 2023-11-12T09:27:48,583 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:48,586 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(130): Open test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:27:48,586 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7254): Opening region: {ENCODED => 3ca9201e34d84739ee34b92f9445e4a2, NAME => 'test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:27:48,587 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:27:48,587 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1699781182417 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:27:48,587 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(888): Instantiated test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:27:48,587 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7296): checking encryption for 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:27:48,587 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7299): checking classloading for 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:27:48,588 INFO [StoreOpener-3ca9201e34d84739ee34b92f9445e4a2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:27:48,589 INFO [StoreOpener-3ca9201e34d84739ee34b92f9445e4a2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ca9201e34d84739ee34b92f9445e4a2 columnFamilyName f 2023-11-12T09:27:48,589 DEBUG [StoreOpener-3ca9201e34d84739ee34b92f9445e4a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:27:48,590 INFO [StoreOpener-3ca9201e34d84739ee34b92f9445e4a2-1 {}] regionserver.HStore(324): Store=3ca9201e34d84739ee34b92f9445e4a2/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:27:48,591 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:27:48,591 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:27:48,593 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1076): writing seq id for 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:27:48,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:27:48,595 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): Opened 3ca9201e34d84739ee34b92f9445e4a2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9740376160, jitterRate=-0.09285677969455719}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:27:48,595 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(992): Region open journal for 3ca9201e34d84739ee34b92f9445e4a2: 2023-11-12T09:27:48,596 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2., pid=88, masterSystemTime=1699781268583 2023-11-12T09:27:48,597 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:27:48,597 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(158): Opened test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:27:48,598 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=87 updating hbase:meta row=3ca9201e34d84739ee34b92f9445e4a2, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:48,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=88, resume processing ppid=87 2023-11-12T09:27:48,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=88, ppid=87, state=SUCCESS; OpenRegionProcedure 3ca9201e34d84739ee34b92f9445e4a2, server=jenkins-hbase3.apache.org,36287,1699781172353 in 167 msec 2023-11-12T09:27:48,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=87, resume processing ppid=86 2023-11-12T09:27:48,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=87, ppid=86, state=SUCCESS; TransitRegionStateProcedure table=test-1699781182417, region=3ca9201e34d84739ee34b92f9445e4a2, ASSIGN in 325 msec 2023-11-12T09:27:48,602 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781182417","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781268602"}]},"ts":"1699781268602"} 2023-11-12T09:27:48,603 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781182417, state=ENABLED in hbase:meta 2023-11-12T09:27:48,606 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'test-1699781182417' completed 2023-11-12T09:27:48,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=86, state=SUCCESS; TruncateTableProcedure (table=test-1699781182417 preserveSplits=true) in 1.1800 sec 2023-11-12T09:27:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=86 2023-11-12T09:27:49,540 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34955/backupUT/backup_1699781257621/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c from hbase tables test-1699781182417 to tables test-1699781182417 2023-11-12T09:27:49,540 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1699781182417 into test-1699781182417 2023-11-12T09:27:49,542 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781269540 from hdfs://localhost:34955/backupUT/backup_1699781257621/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c 2023-11-12T09:27:49,547 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7422b2f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:49,548 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:49,549 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38120, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:49,550 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:27:49,550 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1699781182417 2023-11-12T09:27:49,554 DEBUG [hconnection-0x19559856-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:49,556 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43242, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:49,558 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1699781182417: 2023-11-12T09:27:49,558 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-12T09:27:49,558 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/partitions_bbeeea0d-b53f-462a-96d8-73c89ac8347f 2023-11-12T09:27:49,965 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:49,966 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:49,966 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:49,966 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,941 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-3645715277746830254.jar 2023-11-12T09:27:50,941 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,942 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,942 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,943 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,943 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,943 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,944 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,944 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,944 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,945 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:27:50,946 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:27:50,946 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:27:50,946 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:27:50,947 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:27:50,947 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:27:50,948 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:27:50,948 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:27:50,948 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:27:50,949 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:27:50,949 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:27:50,950 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:27:50,950 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,951 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,951 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,951 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,951 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,952 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:27:50,952 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:27:50,953 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: test-1699781182417 2023-11-12T09:27:50,953 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:50,953 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-12T09:27:50,953 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-12T09:27:50,954 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:27:50,966 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-12T09:27:51,049 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781271011/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:27:51,049 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781271011/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:27:51,097 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34955/backupUT/backup_1699781257621/default/test-1699781182417/archive/data/default/test-1699781182417/15ea62ac691e09a3256d5ddcb7e3267c/f/54153393af664345b273ebc6f013bac7_SeqId_4_ 2023-11-12T09:27:51,102 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-12T09:27:51,133 DEBUG [pool-332-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4431875f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:51,135 DEBUG [hconnection-0x4e47a384-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:51,137 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43258, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:51,141 DEBUG [pool-332-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:51,141 DEBUG [pool-332-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-12T09:27:51,141 DEBUG [pool-332-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:27:51,148 DEBUG [pool-332-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781269540/_temporary/0/_temporary/attempt_local422211408_0011_r_000000_0/f/67a0065212db43ac883b4c8f981b8ff7 is 37, key is row10/f:q1/1699781185078/Put/seqid=0 2023-11-12T09:27:51,150 WARN [Thread-1397 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36287] Specified favored nodes: [jenkins-hbase3.apache.org:36287] 2023-11-12T09:27:52,093 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781269540 2023-11-12T09:27:52,098 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ea8dc8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:52,100 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:52,101 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43272, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:52,106 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781269540/_SUCCESS 2023-11-12T09:27:52,110 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:52,111 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38124, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:52,120 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781269540/f/67a0065212db43ac883b4c8f981b8ff7 first=Optional[row0] last=Optional[row98] 2023-11-12T09:27:52,121 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2., hostname=jenkins-hbase3.apache.org,36287,1699781172353, seqNum=2 for row with hfile group [{f,hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781269540/f/67a0065212db43ac883b4c8f981b8ff7}] 2023-11-12T09:27:52,128 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781269540/f/67a0065212db43ac883b4c8f981b8ff7 for inclusion in 3ca9201e34d84739ee34b92f9445e4a2/f 2023-11-12T09:27:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-12T09:27:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-12T09:27:52,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegion(2520): Flush status journal for 3ca9201e34d84739ee34b92f9445e4a2: 2023-11-12T09:27:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-test-1699781182417-1699781269540/f/67a0065212db43ac883b4c8f981b8ff7 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__test-1699781182417__m70emphgb85nqp74ibg3fkt936a6cinq58ltl74e1a2p6m1no6tplnu36v7ag4i3/f/67a0065212db43ac883b4c8f981b8ff7 2023-11-12T09:27:52,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__test-1699781182417__m70emphgb85nqp74ibg3fkt936a6cinq58ltl74e1a2p6m1no6tplnu36v7ag4i3/f/67a0065212db43ac883b4c8f981b8ff7 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_ 2023-11-12T09:27:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63373f5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:52,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:52,144 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38134, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:52,148 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43282, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(428): write bulk load descriptor to backup test-1699781182417 with 1 entries 2023-11-12T09:27:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(1602): writing raw bulk path hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_ for test-1699781182417 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:27:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(435): written 1 rows for bulk load of test-1699781182417 2023-11-12T09:27:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:52,164 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__test-1699781182417__m70emphgb85nqp74ibg3fkt936a6cinq58ltl74e1a2p6m1no6tplnu36v7ag4i3/f/67a0065212db43ac883b4c8f981b8ff7 into 3ca9201e34d84739ee34b92f9445e4a2/f as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_ - updating store file list. 2023-11-12T09:27:52,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_ into 3ca9201e34d84739ee34b92f9445e4a2/f 2023-11-12T09:27:52,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__test-1699781182417__m70emphgb85nqp74ibg3fkt936a6cinq58ltl74e1a2p6m1no6tplnu36v7ag4i3/f/67a0065212db43ac883b4c8f981b8ff7 into 3ca9201e34d84739ee34b92f9445e4a2/f (new location: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_) 2023-11-12T09:27:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__test-1699781182417__m70emphgb85nqp74ibg3fkt936a6cinq58ltl74e1a2p6m1no6tplnu36v7ag4i3/f/67a0065212db43ac883b4c8f981b8ff7 2023-11-12T09:27:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b93334d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:52,179 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38146, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:27:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:52,185 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43290, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-12T09:27:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup test-1699781182417 with 1 entries 2023-11-12T09:27:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(1523): writing done bulk path hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_ for test-1699781182417 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:27:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(414): written 1 rows for bulk load of test-1699781182417 2023-11-12T09:27:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:52,202 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:52,202 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-12T09:27:52,202 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:27:52,202 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(233): Backup: backup_1699781257621 hdfs://localhost:34955/backupUT/backup_1699781257621/default/test-1699781182417/ 2023-11-12T09:27:52,202 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-12T09:27:52,202 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:52,223 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleOverwriteCommand Thread=605 (was 595) - Thread LEAK? -, OpenFileDescriptor=720 (was 702) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=122 (was 146), ProcessCount=168 (was 168), AvailableMemoryMB=5271 (was 5307) 2023-11-12T09:27:52,223 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-12T09:27:52,241 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleDNECommand Thread=605, OpenFileDescriptor=720, MaxFileDescriptor=60000, SystemLoadAverage=122, ProcessCount=168, AvailableMemoryMB=5270 2023-11-12T09:27:52,241 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-12T09:27:52,241 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(322): test restore fails on multiple tables that do not exist: command-line 2023-11-12T09:27:52,248 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7147cc34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:52,251 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:52,253 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43292, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:52,255 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:27:52,256 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38152, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:27:52,268 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:27:52,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:27:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781272278 (current time:1699781272278). 2023-11-12T09:27:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:27:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:27:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7310945b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:52,288 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43304, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:27:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=89, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:27:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 89 2023-11-12T09:27:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-12T09:27:52,294 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:27:52,295 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:27:52,298 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:27:52,301 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:27:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-12T09:27:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-12T09:27:52,716 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:27:52,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0}] 2023-11-12T09:27:52,718 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:52,838 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:27:52,869 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:52,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=90 2023-11-12T09:27:52,870 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:27:52,870 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-12T09:27:52,874 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/f82d4a97dcba4ae5b07cff1d7ec7effa is 169, key is trslm:hdfs://localhost:34955/backupUT\x00test-1699781182417/meta:log-roll-map/1699781266651/Put/seqid=0 2023-11-12T09:27:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-12T09:27:53,279 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/f82d4a97dcba4ae5b07cff1d7ec7effa 2023-11-12T09:27:53,287 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/0c9e5bc5cf094ea981f2054c8ef12fc3 is 452, key is session:backup_1699781257621/session:context/1699781267074/Put/seqid=0 2023-11-12T09:27:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-12T09:27:53,692 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/0c9e5bc5cf094ea981f2054c8ef12fc3 2023-11-12T09:27:53,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/f82d4a97dcba4ae5b07cff1d7ec7effa as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa 2023-11-12T09:27:53,702 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa, entries=4, sequenceid=72, filesize=5.5 K 2023-11-12T09:27:53,703 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/0c9e5bc5cf094ea981f2054c8ef12fc3 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3 2023-11-12T09:27:53,707 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3, entries=2, sequenceid=72, filesize=5.4 K 2023-11-12T09:27:53,708 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 838ms, sequenceid=72, compaction requested=true 2023-11-12T09:27:53,708 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(2520): Flush status journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:27:53,708 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. for snapshot_backup_system completed. 2023-11-12T09:27:53,708 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:27:53,708 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:27:53,708 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa] hfiles 2023-11-12T09:27:53,708 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db for snapshot=snapshot_backup_system 2023-11-12T09:27:53,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81 for snapshot=snapshot_backup_system 2023-11-12T09:27:53,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc for snapshot=snapshot_backup_system 2023-11-12T09:27:53,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd for snapshot=snapshot_backup_system 2023-11-12T09:27:53,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00 for snapshot=snapshot_backup_system 2023-11-12T09:27:53,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa for snapshot=snapshot_backup_system 2023-11-12T09:27:53,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3] hfiles 2023-11-12T09:27:53,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 for snapshot=snapshot_backup_system 2023-11-12T09:27:53,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b for snapshot=snapshot_backup_system 2023-11-12T09:27:53,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd for snapshot=snapshot_backup_system 2023-11-12T09:27:53,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c for snapshot=snapshot_backup_system 2023-11-12T09:27:53,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5 for snapshot=snapshot_backup_system 2023-11-12T09:27:53,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8 for snapshot=snapshot_backup_system 2023-11-12T09:27:53,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3 for snapshot=snapshot_backup_system 2023-11-12T09:27:54,114 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:27:54,114 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=90 2023-11-12T09:27:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=90 2023-11-12T09:27:54,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:54,116 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:27:54,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=90, resume processing ppid=89 2023-11-12T09:27:54,118 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:27:54,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=90, ppid=89, state=SUCCESS; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 in 1.4010 sec 2023-11-12T09:27:54,119 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:27:54,119 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:27:54,119 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:27:54,120 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:27:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-12T09:27:54,533 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:27:54,543 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:27:54,544 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:27:54,545 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:27:54,545 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 89 2023-11-12T09:27:54,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2560 sec 2023-11-12T09:27:56,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-12T09:27:56,398 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(115): Backup backup_1699781272248 started at 1699781276397. 2023-11-12T09:27:56,399 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(119): Backup session backup_1699781272248 has been started. 2023-11-12T09:27:56,404 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:27:56,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:27:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:27:56,405 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:27:56,405 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:27:56,406 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:27:56,406 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:27:56,406 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:56,408 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:56,408 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:27:56,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:27:56,408 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:27:56,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:56,409 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:27:56,409 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:27:56,409 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:27:56,409 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:27:56,409 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:27:56,409 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:27:56,409 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:27:56,410 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:27:56,410 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:27:56,410 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:27:56,413 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:56,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:56,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:56,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:56,413 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:27:56,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:56,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:56,414 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:56,414 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:56,414 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:56,415 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:27:56,415 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3b92a870[Count = 0] remaining members to acquire global barrier 2023-11-12T09:27:56,415 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:27:56,415 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:56,416 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:56,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:27:56,416 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:27:56,416 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:27:56,416 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:27:56,416 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:56,417 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781261784 highest: 1699781261784 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:56,417 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781261784) roll requested 2023-11-12T09:27:56,429 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781276417, exclude list is [], retry=0 2023-11-12T09:27:56,433 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:27:56,435 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781261784 with entries=15, filesize=4.76 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781276417 2023-11-12T09:27:56,435 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:27:56,435 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781261784 is not closed yet, will try archiving it next time 2023-11-12T09:27:56,435 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781261803) roll requested 2023-11-12T09:27:56,435 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781247003 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781247003 2023-11-12T09:27:56,447 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781276435.meta, exclude list is [], retry=0 2023-11-12T09:27:56,450 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:27:56,451 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781261803.meta with entries=12, filesize=3.07 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781276435.meta 2023-11-12T09:27:56,451 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:27:56,451 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781261803.meta is not closed yet, will try archiving it next time 2023-11-12T09:27:56,457 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:27:56,457 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781276417 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:56,464 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:27:56,464 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36287,1699781172353' in zk 2023-11-12T09:27:56,468 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:27:56,468 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:27:56,468 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:27:56,468 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:56,468 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:56,468 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:56,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:56,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:56,469 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:56,470 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:56,470 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:56,470 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:56,470 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:56,471 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36287,1699781172353': 2023-11-12T09:27:56,471 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:27:56,471 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:27:56,471 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:27:56,471 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:27:56,471 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:27:56,471 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:27:56,473 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:56,473 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:27:56,473 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:27:56,474 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:27:56,474 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:56,474 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:27:56,474 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:27:56,474 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:27:56,475 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:56,475 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:27:56,475 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:27:56,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:56,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:56,477 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:27:56,477 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:27:56,477 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:56,481 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:27:56,481 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:27:56,482 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:27:56,482 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:27:56,482 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:27:56,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:27:56,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:27:56,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:27:56,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } 2023-11-12T09:27:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781276587 (current time:1699781276587). 2023-11-12T09:27:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:27:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781276586_ns2_test-16997811824171 VERSION not specified, setting to 2 2023-11-12T09:27:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56bbcb71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:56,595 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42094, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:27:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=91, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } 2023-11-12T09:27:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 }, snapshot procedure id = 91 2023-11-12T09:27:56,599 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:27:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=91 2023-11-12T09:27:56,600 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:27:56,602 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781276586_ns2_test-16997811824171 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:27:56,604 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:27:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=91 2023-11-12T09:27:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=91 2023-11-12T09:27:57,010 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:27:57,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95}] 2023-11-12T09:27:57,012 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:27:57,163 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:57,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=92 2023-11-12T09:27:57,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:27:57,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.HRegion(2520): Flush status journal for 6e68e35ec82e3efaab2edd4a90156b95: 2023-11-12T09:27:57,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. for snapshot_1699781276586_ns2_test-16997811824171 completed. 2023-11-12T09:27:57,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95.' region-info for snapshot=snapshot_1699781276586_ns2_test-16997811824171 2023-11-12T09:27:57,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:27:57,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_] hfiles 2023-11-12T09:27:57,164 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ for snapshot=snapshot_1699781276586_ns2_test-16997811824171 2023-11-12T09:27:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=91 2023-11-12T09:27:57,570 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:27:57,570 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=92 2023-11-12T09:27:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=92 2023-11-12T09:27:57,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781276586_ns2_test-16997811824171 on region 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:27:57,572 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:27:57,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=92, resume processing ppid=91 2023-11-12T09:27:57,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=92, ppid=91, state=SUCCESS; SnapshotRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95 in 562 msec 2023-11-12T09:27:57,574 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:27:57,574 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:27:57,575 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:27:57,575 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781276586_ns2_test-16997811824171 2023-11-12T09:27:57,575 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781276586_ns2_test-16997811824171 2023-11-12T09:27:57,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=91 2023-11-12T09:27:57,985 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:27:57,990 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:27:57,991 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781276586_ns2_test-16997811824171 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781276586_ns2_test-16997811824171 2023-11-12T09:27:57,992 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:27:57,992 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 }, snapshot procedure id = 91 2023-11-12T09:27:57,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1699781276586_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } in 1.3950 sec 2023-11-12T09:27:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=91 2023-11-12T09:27:58,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } 2023-11-12T09:27:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781278706 (current time:1699781278706). 2023-11-12T09:27:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:27:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781278706_ns3_test-16997811824172 VERSION not specified, setting to 2 2023-11-12T09:27:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6af2a3b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:27:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:27:58,716 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:42102, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:27:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:27:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:27:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=93, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } 2023-11-12T09:27:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 }, snapshot procedure id = 93 2023-11-12T09:27:58,720 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:27:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2023-11-12T09:27:58,720 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:27:58,723 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781278706_ns3_test-16997811824172 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:27:58,725 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:27:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2023-11-12T09:27:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2023-11-12T09:27:59,133 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:27:59,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure 8e08148d1dda03a07c8babda3e33ae17}] 2023-11-12T09:27:59,135 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:27:59,286 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:27:59,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=94 2023-11-12T09:27:59,287 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:27:59,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.HRegion(2520): Flush status journal for 8e08148d1dda03a07c8babda3e33ae17: 2023-11-12T09:27:59,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. for snapshot_1699781278706_ns3_test-16997811824172 completed. 2023-11-12T09:27:59,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17.' region-info for snapshot=snapshot_1699781278706_ns3_test-16997811824172 2023-11-12T09:27:59,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:27:59,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-12T09:27:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2023-11-12T09:27:59,695 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:27:59,695 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=94 2023-11-12T09:27:59,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=94 2023-11-12T09:27:59,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781278706_ns3_test-16997811824172 on region 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:27:59,697 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:27:59,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=94, resume processing ppid=93 2023-11-12T09:27:59,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=94, ppid=93, state=SUCCESS; SnapshotRegionProcedure 8e08148d1dda03a07c8babda3e33ae17 in 564 msec 2023-11-12T09:27:59,699 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:27:59,699 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:27:59,700 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:27:59,700 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781278706_ns3_test-16997811824172 2023-11-12T09:27:59,701 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781278706_ns3_test-16997811824172 2023-11-12T09:27:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2023-11-12T09:28:00,111 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:28:00,115 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:28:00,116 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781278706_ns3_test-16997811824172 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781278706_ns3_test-16997811824172 2023-11-12T09:28:00,117 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:28:00,117 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 }, snapshot procedure id = 93 2023-11-12T09:28:00,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=93, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1699781278706_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } in 1.4000 sec 2023-11-12T09:28:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2023-11-12T09:28:00,825 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781272248 2023-11-12T09:28:00,825 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:28:00,825 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-12T09:28:00,825 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781272248_ns3:test-16997811824172 2023-11-12T09:28:00,825 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781278706_ns3_test-16997811824172 to hdfs://localhost:34955/backupUT/backup_1699781272248/ns3/test-16997811824172/ 2023-11-12T09:28:00,825 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:28:00,854 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:28:00,855 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_1542529642_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781272248/ns3/test-16997811824172, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781272248/ns3/test-16997811824172/.hbase-snapshot/.tmp/snapshot_1699781278706_ns3_test-16997811824172 2023-11-12T09:28:00,855 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781278706_ns3_test-16997811824172. 2023-11-12T09:28:00,864 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781278706_ns3_test-16997811824172 to hdfs://localhost:34955/backupUT/backup_1699781272248/ns3/test-16997811824172/.hbase-snapshot/.tmp/snapshot_1699781278706_ns3_test-16997811824172 2023-11-12T09:28:01,282 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:01,282 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:01,282 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:01,283 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:02,236 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-2248528150609807762.jar 2023-11-12T09:28:02,236 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:02,237 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:02,237 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:02,237 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:02,238 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:02,238 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:02,238 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:02,239 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:02,239 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:02,240 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:28:02,240 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:28:02,241 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:28:02,241 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:28:02,241 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:28:02,242 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:28:02,242 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:28:02,243 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:28:02,243 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:28:02,243 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:28:02,244 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:28:02,244 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:28:02,245 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:02,245 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:02,246 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:02,246 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:02,246 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:02,247 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:02,248 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:02,290 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781278706_ns3_test-16997811824172' hfile list 2023-11-12T09:28:02,361 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781282327/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:28:02,361 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781282327/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:28:03,389 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:28:03,390 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:28:03,395 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781278706_ns3_test-16997811824172 2023-11-12T09:28:03,395 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781278706_ns3_test-16997811824172 finished. 2023-11-12T09:28:03,395 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781272248_ns2:test-16997811824171 2023-11-12T09:28:03,395 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781276586_ns2_test-16997811824171 to hdfs://localhost:34955/backupUT/backup_1699781272248/ns2/test-16997811824171/ 2023-11-12T09:28:03,395 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:28:03,419 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:28:03,419 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_-1524949610_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781272248/ns2/test-16997811824171, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781272248/ns2/test-16997811824171/.hbase-snapshot/.tmp/snapshot_1699781276586_ns2_test-16997811824171 2023-11-12T09:28:03,419 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781276586_ns2_test-16997811824171. 2023-11-12T09:28:03,427 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781276586_ns2_test-16997811824171 to hdfs://localhost:34955/backupUT/backup_1699781272248/ns2/test-16997811824171/.hbase-snapshot/.tmp/snapshot_1699781276586_ns2_test-16997811824171 2023-11-12T09:28:03,850 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:03,850 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:03,851 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:03,851 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:04,847 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-8342165631079640873.jar 2023-11-12T09:28:04,848 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:04,848 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:04,849 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:04,849 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:04,849 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:04,850 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:04,850 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:04,850 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:04,850 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:04,851 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:28:04,852 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:28:04,852 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:28:04,853 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:28:04,853 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:28:04,853 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:28:04,854 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:28:04,854 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:28:04,855 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:28:04,855 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:28:04,855 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:28:04,856 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:28:04,856 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:04,857 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:04,857 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:04,858 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:04,858 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:04,858 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:04,859 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:04,901 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781276586_ns2_test-16997811824171' hfile list 2023-11-12T09:28:04,903 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-12T09:28:04,998 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781284962/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:28:04,998 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781284962/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:28:05,054 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:28:05,077 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ output=hdfs://localhost:34955/backupUT/backup_1699781272248/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ 2023-11-12T09:28:05,077 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-11-12T09:28:06,032 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:28:06,033 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:28:06,041 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781276586_ns2_test-16997811824171 2023-11-12T09:28:06,041 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781276586_ns2_test-16997811824171 finished. 2023-11-12T09:28:06,041 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): ns3:test-16997811824172 2023-11-12T09:28:06,041 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): ns2:test-16997811824171 2023-11-12T09:28:06,049 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781272248 2023-11-12T09:28:06,049 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:28:06,456 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781272248/.backup.manifest 2023-11-12T09:28:06,456 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781272248 2023-11-12T09:28:06,456 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:28:06,862 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781272248/.backup.manifest 2023-11-12T09:28:06,862 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(358): Backup backup_1699781272248 finished: type=FULL,tablelist=ns3:test-16997811824172;ns2:test-16997811824171,targetRootDir=hdfs://localhost:34955/backupUT,startts=1699781276397,completets=1699781286049,bytescopied=0 2023-11-12T09:28:06,862 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:28:06,862 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781278706_ns3_test-16997811824172 2023-11-12T09:28:06,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781278706_ns3_test-16997811824172" 2023-11-12T09:28:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781278706_ns3_test-16997811824172 2023-11-12T09:28:06,866 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781278706_ns3_test-16997811824172 for backup backup_1699781272248 succeeded. 2023-11-12T09:28:06,866 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781276586_ns2_test-16997811824171 2023-11-12T09:28:06,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781276586_ns2_test-16997811824171" 2023-11-12T09:28:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781276586_ns2_test-16997811824171 2023-11-12T09:28:06,871 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781276586_ns2_test-16997811824171 for backup backup_1699781272248 succeeded. 2023-11-12T09:28:06,872 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:28:06,876 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:28:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:28:06,878 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:28:06,880 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:28:06,885 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(376): Backup backup_1699781272248 completed. 2023-11-12T09:28:06,886 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client You have to specify either set name or table list to restore Usage: hbase restore [options] backup_path Path to a backup destination root backup_id Backup image ID to restore table(s) Comma-separated list of tables to restore Options: -c Check restore sequence and dependencies only (does not execute the command) -d Enable debug loggings -h,--help Show usage -m A comma separated list of target tables. If specified, each table in must have a mapping -o Overwrite data if any of the restore target tables exists -q Yarn queue name to run backup restore command on -s Backup set to restore, mutually exclusive with -t (table list) -t Table name list, comma-separated. To enable backup, in hbase-site.xml, set: hbase.backup.enable=true hbase.master.logcleaner.plugins=YOUR_PLUGINS,org.apache.hadoop.hbase.backup.master.BackupLogCleaner hbase.procedure.master.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager hbase.procedure.regionserver.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager hbase.coprocessor.region.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.BackupObserver and restart the cluster For more information please see http://hbase.apache.org/book.html#backuprestore 2023-11-12T09:28:06,914 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleDNECommand Thread=598 (was 605), OpenFileDescriptor=711 (was 720), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=122 (was 122), ProcessCount=171 (was 168) - ProcessCount LEAK? -, AvailableMemoryMB=5236 (was 5270) 2023-11-12T09:28:06,914 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2023-11-12T09:28:06,929 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultiple Thread=598, OpenFileDescriptor=711, MaxFileDescriptor=60000, SystemLoadAverage=122, ProcessCount=171, AvailableMemoryMB=5236 2023-11-12T09:28:06,929 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2023-11-12T09:28:06,929 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(120): create full backup image on multiple tables 2023-11-12T09:28:06,935 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca747e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:06,938 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:06,940 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:37566, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:06,942 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:06,943 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38926, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:28:06,955 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:28:06,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:28:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781286960 (current time:1699781286960). 2023-11-12T09:28:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:28:06,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:28:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c299620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:06,977 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:37576, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:06,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:06,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:28:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=95, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:28:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 95 2023-11-12T09:28:06,983 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:28:06,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-12T09:28:06,984 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:28:06,986 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:28:06,989 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:28:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-12T09:28:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-12T09:28:07,395 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:28:07,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0}] 2023-11-12T09:28:07,397 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:28:07,548 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:07,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=96 2023-11-12T09:28:07,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:28:07,549 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-11-12T09:28:07,553 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/84fb173346f74b8e9ff8d878b5974b87 is 171, key is trslm:hdfs://localhost:34955/backupUT\x00ns2:test-16997811824171/meta:log-roll-map/1699781286044/Put/seqid=0 2023-11-12T09:28:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-12T09:28:07,960 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/84fb173346f74b8e9ff8d878b5974b87 2023-11-12T09:28:07,968 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/4c19f00d68984382ac24c49b4f6d0244 is 524, key is session:backup_1699781272248/session:context/1699781286879/Put/seqid=0 2023-11-12T09:28:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-12T09:28:08,372 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/4c19f00d68984382ac24c49b4f6d0244 2023-11-12T09:28:08,378 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/84fb173346f74b8e9ff8d878b5974b87 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87 2023-11-12T09:28:08,382 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87, entries=6, sequenceid=83, filesize=5.8 K 2023-11-12T09:28:08,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/4c19f00d68984382ac24c49b4f6d0244 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244 2023-11-12T09:28:08,387 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244, entries=2, sequenceid=83, filesize=5.5 K 2023-11-12T09:28:08,388 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 839ms, sequenceid=83, compaction requested=true 2023-11-12T09:28:08,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(2520): Flush status journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:28:08,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. for snapshot_backup_system completed. 2023-11-12T09:28:08,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:28:08,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:28:08,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87] hfiles 2023-11-12T09:28:08,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db for snapshot=snapshot_backup_system 2023-11-12T09:28:08,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81 for snapshot=snapshot_backup_system 2023-11-12T09:28:08,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc for snapshot=snapshot_backup_system 2023-11-12T09:28:08,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd for snapshot=snapshot_backup_system 2023-11-12T09:28:08,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00 for snapshot=snapshot_backup_system 2023-11-12T09:28:08,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa for snapshot=snapshot_backup_system 2023-11-12T09:28:08,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87 for snapshot=snapshot_backup_system 2023-11-12T09:28:08,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244] hfiles 2023-11-12T09:28:08,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 for snapshot=snapshot_backup_system 2023-11-12T09:28:08,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b for snapshot=snapshot_backup_system 2023-11-12T09:28:08,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd for snapshot=snapshot_backup_system 2023-11-12T09:28:08,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c for snapshot=snapshot_backup_system 2023-11-12T09:28:08,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5 for snapshot=snapshot_backup_system 2023-11-12T09:28:08,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8 for snapshot=snapshot_backup_system 2023-11-12T09:28:08,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3 for snapshot=snapshot_backup_system 2023-11-12T09:28:08,389 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244 for snapshot=snapshot_backup_system 2023-11-12T09:28:08,794 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:28:08,794 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=96 2023-11-12T09:28:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=96 2023-11-12T09:28:08,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 606338a328e82acae51355d485ed81f0 2023-11-12T09:28:08,798 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:28:08,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=96, resume processing ppid=95 2023-11-12T09:28:08,800 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:28:08,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=96, ppid=95, state=SUCCESS; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 in 1.4040 sec 2023-11-12T09:28:08,801 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:28:08,801 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:28:08,801 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:28:08,802 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:28:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-12T09:28:09,212 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:28:09,222 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:28:09,223 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:28:09,224 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:28:09,224 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 95 2023-11-12T09:28:09,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=95, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2460 sec 2023-11-12T09:28:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-12T09:28:11,092 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(115): Backup backup_1699781286935 started at 1699781291092. 2023-11-12T09:28:11,094 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(119): Backup session backup_1699781286935 has been started. 2023-11-12T09:28:11,098 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:28:11,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:28:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:28:11,099 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:28:11,099 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:28:11,100 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:28:11,100 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:28:11,101 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:28:11,102 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:11,102 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:28:11,102 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:28:11,103 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:28:11,103 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:28:11,104 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:28:11,104 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:28:11,104 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:28:11,104 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:28:11,104 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:28:11,104 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:28:11,104 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:28:11,105 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:28:11,105 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:28:11,105 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:28:11,108 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:28:11,108 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:11,108 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:28:11,108 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:28:11,108 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:28:11,108 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:28:11,109 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:28:11,109 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:11,109 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:11,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:28:11,110 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:28:11,110 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6813c263[Count = 0] remaining members to acquire global barrier 2023-11-12T09:28:11,110 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:28:11,110 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:28:11,112 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:28:11,112 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:28:11,112 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:28:11,112 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:28:11,112 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:28:11,112 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:11,112 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781276417 highest: 1699781276417 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:11,112 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781276417) roll requested 2023-11-12T09:28:11,123 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781291112, exclude list is [], retry=0 2023-11-12T09:28:11,129 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:28:11,130 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781276417 with entries=10, filesize=2.67 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781291112 2023-11-12T09:28:11,130 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:28:11,131 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781276417 is not closed yet, will try archiving it next time 2023-11-12T09:28:11,131 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781276435) roll requested 2023-11-12T09:28:11,142 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781291131.meta, exclude list is [], retry=0 2023-11-12T09:28:11,145 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:28:11,146 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781276435.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781291131.meta 2023-11-12T09:28:11,147 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:28:11,147 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781276435.meta is not closed yet, will try archiving it next time 2023-11-12T09:28:11,149 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781276435.meta not finished, retry = 0 2023-11-12T09:28:11,152 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:28:11,152 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781291112 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:11,163 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:28:11,163 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36287,1699781172353' in zk 2023-11-12T09:28:11,166 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:28:11,166 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:28:11,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:11,166 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:28:11,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:28:11,166 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:28:11,167 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:28:11,167 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:28:11,167 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:11,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:11,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:28:11,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:11,169 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:11,169 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36287,1699781172353': 2023-11-12T09:28:11,169 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:28:11,169 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:28:11,169 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:28:11,169 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:28:11,169 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:28:11,169 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:28:11,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:28:11,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:28:11,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:28:11,171 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:28:11,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:28:11,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:28:11,172 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:28:11,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:28:11,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:28:11,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:11,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:28:11,173 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:11,173 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:11,174 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:28:11,174 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:11,174 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:11,182 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:28:11,182 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:28:11,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:28:11,182 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:28:11,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:28:11,183 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:28:11,183 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:28:11,250 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781276435.meta to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781276435.meta 2023-11-12T09:28:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:28:11,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } 2023-11-12T09:28:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781291287 (current time:1699781291287). 2023-11-12T09:28:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:28:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781291286_ns2_test-16997811824171 VERSION not specified, setting to 2 2023-11-12T09:28:11,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d5c181e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:11,297 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:37584, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:28:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=97, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } 2023-11-12T09:28:11,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 }, snapshot procedure id = 97 2023-11-12T09:28:11,300 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:28:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=97 2023-11-12T09:28:11,301 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:28:11,303 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781291286_ns2_test-16997811824171 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:28:11,306 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:28:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=97 2023-11-12T09:28:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=97 2023-11-12T09:28:11,711 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:28:11,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95}] 2023-11-12T09:28:11,714 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:28:11,865 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:11,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=98 2023-11-12T09:28:11,865 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:28:11,866 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.HRegion(2520): Flush status journal for 6e68e35ec82e3efaab2edd4a90156b95: 2023-11-12T09:28:11,866 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. for snapshot_1699781291286_ns2_test-16997811824171 completed. 2023-11-12T09:28:11,866 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95.' region-info for snapshot=snapshot_1699781291286_ns2_test-16997811824171 2023-11-12T09:28:11,866 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:28:11,866 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_] hfiles 2023-11-12T09:28:11,866 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ for snapshot=snapshot_1699781291286_ns2_test-16997811824171 2023-11-12T09:28:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=97 2023-11-12T09:28:12,273 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:28:12,273 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=98 2023-11-12T09:28:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=98 2023-11-12T09:28:12,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781291286_ns2_test-16997811824171 on region 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:28:12,275 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:28:12,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=98, resume processing ppid=97 2023-11-12T09:28:12,277 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:28:12,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=98, ppid=97, state=SUCCESS; SnapshotRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95 in 564 msec 2023-11-12T09:28:12,277 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:28:12,278 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:28:12,278 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781291286_ns2_test-16997811824171 2023-11-12T09:28:12,278 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781291286_ns2_test-16997811824171 2023-11-12T09:28:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=97 2023-11-12T09:28:12,687 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:28:12,692 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:28:12,693 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781291286_ns2_test-16997811824171 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781291286_ns2_test-16997811824171 2023-11-12T09:28:12,694 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:28:12,694 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 }, snapshot procedure id = 97 2023-11-12T09:28:12,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=97, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1699781291286_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } in 1.3960 sec 2023-11-12T09:28:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=97 2023-11-12T09:28:13,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } 2023-11-12T09:28:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781293407 (current time:1699781293407). 2023-11-12T09:28:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:28:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781293406_ns3_test-16997811824172 VERSION not specified, setting to 2 2023-11-12T09:28:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a36ea68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:13,416 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:37588, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:28:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=99, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } 2023-11-12T09:28:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 }, snapshot procedure id = 99 2023-11-12T09:28:13,420 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:28:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=99 2023-11-12T09:28:13,421 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:28:13,424 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781293406_ns3_test-16997811824172 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:28:13,426 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:28:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=99 2023-11-12T09:28:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=99 2023-11-12T09:28:13,832 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:28:13,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure 8e08148d1dda03a07c8babda3e33ae17}] 2023-11-12T09:28:13,834 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:28:13,985 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:13,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=100 2023-11-12T09:28:13,986 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:28:13,986 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.HRegion(2520): Flush status journal for 8e08148d1dda03a07c8babda3e33ae17: 2023-11-12T09:28:13,986 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. for snapshot_1699781293406_ns3_test-16997811824172 completed. 2023-11-12T09:28:13,986 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17.' region-info for snapshot=snapshot_1699781293406_ns3_test-16997811824172 2023-11-12T09:28:13,986 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:28:13,986 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-12T09:28:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=99 2023-11-12T09:28:14,392 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:28:14,392 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=100 2023-11-12T09:28:14,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=100 2023-11-12T09:28:14,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781293406_ns3_test-16997811824172 on region 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:28:14,394 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:28:14,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=100, resume processing ppid=99 2023-11-12T09:28:14,396 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:28:14,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=100, ppid=99, state=SUCCESS; SnapshotRegionProcedure 8e08148d1dda03a07c8babda3e33ae17 in 563 msec 2023-11-12T09:28:14,397 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:28:14,397 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:28:14,397 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781293406_ns3_test-16997811824172 2023-11-12T09:28:14,399 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781293406_ns3_test-16997811824172 2023-11-12T09:28:14,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=99 2023-11-12T09:28:14,807 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:28:14,812 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:28:14,812 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781293406_ns3_test-16997811824172 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781293406_ns3_test-16997811824172 2023-11-12T09:28:14,813 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:28:14,813 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 }, snapshot procedure id = 99 2023-11-12T09:28:14,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=99, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1699781293406_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } in 1.3950 sec 2023-11-12T09:28:15,135 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.38461538461538464, tune throughput to 69.23 MB/second 2023-11-12T09:28:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=99 2023-11-12T09:28:15,527 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781286935 2023-11-12T09:28:15,527 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:28:15,527 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-12T09:28:15,527 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781286935_ns3:test-16997811824172 2023-11-12T09:28:15,527 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781293406_ns3_test-16997811824172 to hdfs://localhost:34955/backupUT/backup_1699781286935/ns3/test-16997811824172/ 2023-11-12T09:28:15,527 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:28:15,551 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:28:15,551 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_452863238_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781286935/ns3/test-16997811824172, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781286935/ns3/test-16997811824172/.hbase-snapshot/.tmp/snapshot_1699781293406_ns3_test-16997811824172 2023-11-12T09:28:15,551 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781293406_ns3_test-16997811824172. 2023-11-12T09:28:15,556 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781293406_ns3_test-16997811824172 to hdfs://localhost:34955/backupUT/backup_1699781286935/ns3/test-16997811824172/.hbase-snapshot/.tmp/snapshot_1699781293406_ns3_test-16997811824172 2023-11-12T09:28:15,982 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:15,983 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:15,983 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:15,984 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:16,952 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-4421829928835325836.jar 2023-11-12T09:28:16,952 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:16,953 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:16,953 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:16,954 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:16,954 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:16,954 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:16,955 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:16,955 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:16,955 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:16,956 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:28:16,957 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:28:16,957 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:28:16,957 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:28:16,958 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:28:16,958 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:28:16,958 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:28:16,959 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:28:16,959 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:28:16,960 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:28:16,960 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:28:16,960 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:28:16,961 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:16,961 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:16,962 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:16,962 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:16,963 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:16,963 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:16,964 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:17,010 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781293406_ns3_test-16997811824172' hfile list 2023-11-12T09:28:17,082 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781297048/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:28:17,082 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781297048/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:28:18,110 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:28:18,111 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:28:18,116 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781293406_ns3_test-16997811824172 2023-11-12T09:28:18,116 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781293406_ns3_test-16997811824172 finished. 2023-11-12T09:28:18,116 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781286935_ns2:test-16997811824171 2023-11-12T09:28:18,116 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781291286_ns2_test-16997811824171 to hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171/ 2023-11-12T09:28:18,116 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:28:18,152 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:28:18,152 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_-1521732397_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171/.hbase-snapshot/.tmp/snapshot_1699781291286_ns2_test-16997811824171 2023-11-12T09:28:18,152 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781291286_ns2_test-16997811824171. 2023-11-12T09:28:18,160 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781291286_ns2_test-16997811824171 to hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171/.hbase-snapshot/.tmp/snapshot_1699781291286_ns2_test-16997811824171 2023-11-12T09:28:18,576 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:18,576 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:18,577 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:18,577 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:19,558 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-3444717261551532041.jar 2023-11-12T09:28:19,558 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:19,558 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:19,559 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:19,559 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:19,559 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:19,560 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:19,560 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:19,560 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:19,561 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:19,561 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:28:19,562 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:28:19,562 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:28:19,563 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:28:19,563 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:28:19,564 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:28:19,564 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:28:19,564 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:28:19,565 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:28:19,565 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:28:19,565 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:28:19,566 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:28:19,566 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:19,567 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:19,568 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:19,568 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:19,568 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:19,569 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:19,569 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:19,624 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781291286_ns2_test-16997811824171' hfile list 2023-11-12T09:28:19,626 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-12T09:28:19,704 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781299666/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:28:19,704 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781299666/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:28:19,759 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:28:19,783 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ output=hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ 2023-11-12T09:28:19,783 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-11-12T09:28:20,738 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:28:20,739 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:28:20,747 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781291286_ns2_test-16997811824171 2023-11-12T09:28:20,748 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781291286_ns2_test-16997811824171 finished. 2023-11-12T09:28:20,748 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): ns3:test-16997811824172 2023-11-12T09:28:20,748 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): ns2:test-16997811824171 2023-11-12T09:28:20,758 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781286935 2023-11-12T09:28:20,758 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:28:21,165 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781286935/.backup.manifest 2023-11-12T09:28:21,165 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781286935 2023-11-12T09:28:21,165 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:28:21,570 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781286935/.backup.manifest 2023-11-12T09:28:21,570 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(358): Backup backup_1699781286935 finished: type=FULL,tablelist=ns3:test-16997811824172;ns2:test-16997811824171,targetRootDir=hdfs://localhost:34955/backupUT,startts=1699781291092,completets=1699781300758,bytescopied=0 2023-11-12T09:28:21,570 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:28:21,570 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781293406_ns3_test-16997811824172 2023-11-12T09:28:21,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781293406_ns3_test-16997811824172" 2023-11-12T09:28:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781293406_ns3_test-16997811824172 2023-11-12T09:28:21,574 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781293406_ns3_test-16997811824172 for backup backup_1699781286935 succeeded. 2023-11-12T09:28:21,574 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781291286_ns2_test-16997811824171 2023-11-12T09:28:21,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781291286_ns2_test-16997811824171" 2023-11-12T09:28:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781291286_ns2_test-16997811824171 2023-11-12T09:28:21,577 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781291286_ns2_test-16997811824171 for backup backup_1699781286935 succeeded. 2023-11-12T09:28:21,578 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:28:21,582 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:28:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:28:21,585 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:28:21,586 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:28:21,591 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(376): Backup backup_1699781286935 completed. 2023-11-12T09:28:21,592 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:21,602 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-11-12T09:28:21,603 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-11-12T09:28:21,604 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781286935/.backup.manifest 2023-11-12T09:28:21,605 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781286935/.backup.manifest 2023-11-12T09:28:21,606 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781286935/.backup.manifest 2023-11-12T09:28:21,608 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781286935/.backup.manifest 2023-11-12T09:28:21,608 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781286935/.backup.manifest 2023-11-12T09:28:21,610 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781286935/.backup.manifest 2023-11-12T09:28:21,610 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16997811824171' to 'ns2:table2' from full backup image hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171 2023-11-12T09:28:21,616 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(487): Creating target table 'ns2:table2' 2023-11-12T09:28:21,616 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:28:21,617 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f in region [hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95] 2023-11-12T09:28:21,620 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ first=row0 last=row98 2023-11-12T09:28:21,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:28:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:table2 2023-11-12T09:28:21,623 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:28:21,624 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 101 2023-11-12T09:28:21,624 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:28:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2023-11-12T09:28:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2023-11-12T09:28:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2023-11-12T09:28:22,033 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3bddb82ee5b5de8b4c725633578df152, NAME => 'ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:28:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2023-11-12T09:28:22,439 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:22,439 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1668): Closing 3bddb82ee5b5de8b4c725633578df152, disabling compactions & flushes 2023-11-12T09:28:22,439 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152. 2023-11-12T09:28:22,439 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152. 2023-11-12T09:28:22,439 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152. after waiting 0 ms 2023-11-12T09:28:22,439 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152. 2023-11-12T09:28:22,439 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1909): Closed ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152. 2023-11-12T09:28:22,439 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3bddb82ee5b5de8b4c725633578df152: 2023-11-12T09:28:22,440 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:28:22,441 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1699781302440"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781302440"}]},"ts":"1699781302440"} 2023-11-12T09:28:22,442 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:28:22,443 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:28:22,443 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781302443"}]},"ts":"1699781302443"} 2023-11-12T09:28:22,444 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2023-11-12T09:28:22,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=3bddb82ee5b5de8b4c725633578df152, ASSIGN}] 2023-11-12T09:28:22,450 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=3bddb82ee5b5de8b4c725633578df152, ASSIGN 2023-11-12T09:28:22,451 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:table2, region=3bddb82ee5b5de8b4c725633578df152, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:28:22,602 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=102 updating hbase:meta row=3bddb82ee5b5de8b4c725633578df152, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:22,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE; OpenRegionProcedure 3bddb82ee5b5de8b4c725633578df152, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:28:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2023-11-12T09:28:22,755 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:22,758 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(130): Open ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152. 2023-11-12T09:28:22,759 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7254): Opening region: {ENCODED => 3bddb82ee5b5de8b4c725633578df152, NAME => 'ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:22,759 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:28:22,759 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 3bddb82ee5b5de8b4c725633578df152 2023-11-12T09:28:22,759 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(888): Instantiated ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:22,759 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7296): checking encryption for 3bddb82ee5b5de8b4c725633578df152 2023-11-12T09:28:22,759 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7299): checking classloading for 3bddb82ee5b5de8b4c725633578df152 2023-11-12T09:28:22,761 INFO [StoreOpener-3bddb82ee5b5de8b4c725633578df152-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 3bddb82ee5b5de8b4c725633578df152 2023-11-12T09:28:22,762 INFO [StoreOpener-3bddb82ee5b5de8b4c725633578df152-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3bddb82ee5b5de8b4c725633578df152 columnFamilyName f 2023-11-12T09:28:22,762 DEBUG [StoreOpener-3bddb82ee5b5de8b4c725633578df152-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:22,762 INFO [StoreOpener-3bddb82ee5b5de8b4c725633578df152-1 {}] regionserver.HStore(324): Store=3bddb82ee5b5de8b4c725633578df152/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:22,763 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152 2023-11-12T09:28:22,763 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152 2023-11-12T09:28:22,769 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1076): writing seq id for 3bddb82ee5b5de8b4c725633578df152 2023-11-12T09:28:22,771 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:28:22,772 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1093): Opened 3bddb82ee5b5de8b4c725633578df152; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10003340960, jitterRate=-0.06836627423763275}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:28:22,772 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(992): Region open journal for 3bddb82ee5b5de8b4c725633578df152: 2023-11-12T09:28:22,773 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152., pid=103, masterSystemTime=1699781302755 2023-11-12T09:28:22,774 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152. 2023-11-12T09:28:22,774 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(158): Opened ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152. 2023-11-12T09:28:22,775 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=102 updating hbase:meta row=3bddb82ee5b5de8b4c725633578df152, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:22,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=103, resume processing ppid=102 2023-11-12T09:28:22,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=103, ppid=102, state=SUCCESS; OpenRegionProcedure 3bddb82ee5b5de8b4c725633578df152, server=jenkins-hbase3.apache.org,36287,1699781172353 in 172 msec 2023-11-12T09:28:22,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=102, resume processing ppid=101 2023-11-12T09:28:22,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=102, ppid=101, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=3bddb82ee5b5de8b4c725633578df152, ASSIGN in 329 msec 2023-11-12T09:28:22,779 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:28:22,779 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781302779"}]},"ts":"1699781302779"} 2023-11-12T09:28:22,781 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2023-11-12T09:28:22,783 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:28:22,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=101, state=SUCCESS; CreateTableProcedure table=ns2:table2 in 1.1620 sec 2023-11-12T09:28:23,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2023-11-12T09:28:23,734 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95 from hbase tables ns2:test-16997811824171 to tables ns2:table2 2023-11-12T09:28:23,734 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16997811824171 into ns2:table2 2023-11-12T09:28:23,736 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781303734 from hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:28:23,742 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58576be9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:23,743 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:23,744 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:37220, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:28:23,745 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:28:23,745 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:table2 2023-11-12T09:28:23,749 DEBUG [hconnection-0x151e1d23-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:23,750 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:51020, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:23,752 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:table2: 2023-11-12T09:28:23,752 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-12T09:28:23,752 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/partitions_7801bf51-bf79-4655-a1f1-3ec667c60904 2023-11-12T09:28:24,159 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:24,159 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:24,160 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:24,160 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,127 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-3201117928123421940.jar 2023-11-12T09:28:25,127 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,127 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,128 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,128 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,129 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,129 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,129 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,129 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,130 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,131 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:28:25,131 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:28:25,131 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:28:25,132 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:28:25,132 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:28:25,133 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:28:25,133 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:28:25,133 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:28:25,134 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:28:25,134 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:28:25,135 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:28:25,135 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:28:25,136 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,136 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,136 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,137 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,137 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,137 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:25,138 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:25,138 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:table2 2023-11-12T09:28:25,138 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:25,138 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-12T09:28:25,138 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-12T09:28:25,139 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:28:25,152 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-12T09:28:25,243 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781305210/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:28:25,244 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781305210/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:28:25,312 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ 2023-11-12T09:28:25,316 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-12T09:28:25,338 DEBUG [pool-397-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9aa896, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:25,339 DEBUG [hconnection-0x492e0d34-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:25,341 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57316, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:25,342 DEBUG [pool-397-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:25,343 DEBUG [pool-397-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-12T09:28:25,343 DEBUG [pool-397-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:28:25,351 DEBUG [pool-397-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781303734/_temporary/0/_temporary/attempt_local1314441896_0016_r_000000_0/f/4d02103202d744399e88c7f37a35c82e is 37, key is row10/f:q1/1699781187295/Put/seqid=0 2023-11-12T09:28:25,353 WARN [Thread-1716 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36287] Specified favored nodes: [jenkins-hbase3.apache.org:36287] 2023-11-12T09:28:26,314 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781303734 2023-11-12T09:28:26,320 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40fdf4d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:26,321 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:26,323 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57318, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:26,328 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781303734/_SUCCESS 2023-11-12T09:28:26,331 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:26,333 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:55468, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:28:26,341 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781303734/f/4d02103202d744399e88c7f37a35c82e first=Optional[row0] last=Optional[row98] 2023-11-12T09:28:26,342 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152., hostname=jenkins-hbase3.apache.org,36287,1699781172353, seqNum=2 for row with hfile group [{f,hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781303734/f/4d02103202d744399e88c7f37a35c82e}] 2023-11-12T09:28:26,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781303734/f/4d02103202d744399e88c7f37a35c82e for inclusion in 3bddb82ee5b5de8b4c725633578df152/f 2023-11-12T09:28:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-12T09:28:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-12T09:28:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegion(2520): Flush status journal for 3bddb82ee5b5de8b4c725633578df152: 2023-11-12T09:28:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781303734/f/4d02103202d744399e88c7f37a35c82e to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_table2__jn8ccl6qcj2u68b9i2e8o3b5i31j6g26ila4j2jtv1t28fhbbrj4i4dl136tutt2/f/4d02103202d744399e88c7f37a35c82e 2023-11-12T09:28:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_table2__jn8ccl6qcj2u68b9i2e8o3b5i31j6g26ila4j2jtv1t28fhbbrj4i4dl136tutt2/f/4d02103202d744399e88c7f37a35c82e as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152/f/20145e3258b94f36ab1dba1b24bff233_SeqId_4_ 2023-11-12T09:28:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f2853e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:26,364 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:55476, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:28:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:26,371 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57320, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:26,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_table2__jn8ccl6qcj2u68b9i2e8o3b5i31j6g26ila4j2jtv1t28fhbbrj4i4dl136tutt2/f/4d02103202d744399e88c7f37a35c82e into 3bddb82ee5b5de8b4c725633578df152/f as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152/f/20145e3258b94f36ab1dba1b24bff233_SeqId_4_ - updating store file list. 2023-11-12T09:28:26,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152/f/20145e3258b94f36ab1dba1b24bff233_SeqId_4_ into 3bddb82ee5b5de8b4c725633578df152/f 2023-11-12T09:28:26,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_table2__jn8ccl6qcj2u68b9i2e8o3b5i31j6g26ila4j2jtv1t28fhbbrj4i4dl136tutt2/f/4d02103202d744399e88c7f37a35c82e into 3bddb82ee5b5de8b4c725633578df152/f (new location: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152/f/20145e3258b94f36ab1dba1b24bff233_SeqId_4_) 2023-11-12T09:28:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_table2__jn8ccl6qcj2u68b9i2e8o3b5i31j6g26ila4j2jtv1t28fhbbrj4i4dl136tutt2/f/4d02103202d744399e88c7f37a35c82e 2023-11-12T09:28:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34bbca55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:26,409 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:55484, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:28:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:26,415 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57324, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-12T09:28:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:26,432 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:26,432 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-12T09:28:26,432 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:28:26,432 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(233): Backup: backup_1699781286935 hdfs://localhost:34955/backupUT/backup_1699781286935/ns2/test-16997811824171/ 2023-11-12T09:28:26,433 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781286935/.backup.manifest 2023-11-12T09:28:26,435 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781286935/.backup.manifest 2023-11-12T09:28:26,435 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16997811824172' to 'ns3:table3_restore' from full backup image hdfs://localhost:34955/backupUT/backup_1699781286935/ns3/test-16997811824172 2023-11-12T09:28:26,443 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:34955/backupUT/backup_1699781286935/ns3/test-16997811824172/archive/data/ns3/test-16997811824172 does not exists 2023-11-12T09:28:26,443 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16997811824172, will only create table 2023-11-12T09:28:26,444 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(487): Creating target table 'ns3:table3_restore' 2023-11-12T09:28:26,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:28:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:table3_restore 2023-11-12T09:28:26,447 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:28:26,447 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "table3_restore" procId is: 104 2023-11-12T09:28:26,448 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:28:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=104 2023-11-12T09:28:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=104 2023-11-12T09:28:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=104 2023-11-12T09:28:26,854 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 19bae2349079b2c732d5ae3c1c5c9eda, NAME => 'ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:28:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=104 2023-11-12T09:28:27,261 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:27,261 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1668): Closing 19bae2349079b2c732d5ae3c1c5c9eda, disabling compactions & flushes 2023-11-12T09:28:27,261 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda. 2023-11-12T09:28:27,261 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda. 2023-11-12T09:28:27,261 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda. after waiting 0 ms 2023-11-12T09:28:27,261 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda. 2023-11-12T09:28:27,261 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda. 2023-11-12T09:28:27,261 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1622): Region close journal for 19bae2349079b2c732d5ae3c1c5c9eda: 2023-11-12T09:28:27,262 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:28:27,262 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1699781307262"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781307262"}]},"ts":"1699781307262"} 2023-11-12T09:28:27,264 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:28:27,265 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:28:27,265 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781307265"}]},"ts":"1699781307265"} 2023-11-12T09:28:27,266 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLING in hbase:meta 2023-11-12T09:28:27,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=19bae2349079b2c732d5ae3c1c5c9eda, ASSIGN}] 2023-11-12T09:28:27,277 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=19bae2349079b2c732d5ae3c1c5c9eda, ASSIGN 2023-11-12T09:28:27,277 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:table3_restore, region=19bae2349079b2c732d5ae3c1c5c9eda, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:28:27,429 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=105 updating hbase:meta row=19bae2349079b2c732d5ae3c1c5c9eda, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:27,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE; OpenRegionProcedure 19bae2349079b2c732d5ae3c1c5c9eda, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:28:27,551 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2023-11-12T09:28:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=104 2023-11-12T09:28:27,582 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:27,586 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(130): Open ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda. 2023-11-12T09:28:27,586 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7254): Opening region: {ENCODED => 19bae2349079b2c732d5ae3c1c5c9eda, NAME => 'ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:27,587 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:28:27,587 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table3_restore 19bae2349079b2c732d5ae3c1c5c9eda 2023-11-12T09:28:27,587 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:27,587 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7296): checking encryption for 19bae2349079b2c732d5ae3c1c5c9eda 2023-11-12T09:28:27,587 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7299): checking classloading for 19bae2349079b2c732d5ae3c1c5c9eda 2023-11-12T09:28:27,588 INFO [StoreOpener-19bae2349079b2c732d5ae3c1c5c9eda-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 19bae2349079b2c732d5ae3c1c5c9eda 2023-11-12T09:28:27,589 INFO [StoreOpener-19bae2349079b2c732d5ae3c1c5c9eda-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19bae2349079b2c732d5ae3c1c5c9eda columnFamilyName f 2023-11-12T09:28:27,589 DEBUG [StoreOpener-19bae2349079b2c732d5ae3c1c5c9eda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:27,590 INFO [StoreOpener-19bae2349079b2c732d5ae3c1c5c9eda-1 {}] regionserver.HStore(324): Store=19bae2349079b2c732d5ae3c1c5c9eda/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:27,590 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/19bae2349079b2c732d5ae3c1c5c9eda 2023-11-12T09:28:27,591 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/19bae2349079b2c732d5ae3c1c5c9eda 2023-11-12T09:28:27,592 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1076): writing seq id for 19bae2349079b2c732d5ae3c1c5c9eda 2023-11-12T09:28:27,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/19bae2349079b2c732d5ae3c1c5c9eda/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:28:27,594 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): Opened 19bae2349079b2c732d5ae3c1c5c9eda; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11210354240, jitterRate=0.044045597314834595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:28:27,595 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(992): Region open journal for 19bae2349079b2c732d5ae3c1c5c9eda: 2023-11-12T09:28:27,595 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda., pid=106, masterSystemTime=1699781307582 2023-11-12T09:28:27,597 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda. 2023-11-12T09:28:27,597 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(158): Opened ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda. 2023-11-12T09:28:27,597 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=105 updating hbase:meta row=19bae2349079b2c732d5ae3c1c5c9eda, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:27,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=106, resume processing ppid=105 2023-11-12T09:28:27,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=106, ppid=105, state=SUCCESS; OpenRegionProcedure 19bae2349079b2c732d5ae3c1c5c9eda, server=jenkins-hbase3.apache.org,36287,1699781172353 in 169 msec 2023-11-12T09:28:27,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=105, resume processing ppid=104 2023-11-12T09:28:27,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=105, ppid=104, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=19bae2349079b2c732d5ae3c1c5c9eda, ASSIGN in 328 msec 2023-11-12T09:28:27,602 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:28:27,602 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781307602"}]},"ts":"1699781307602"} 2023-11-12T09:28:27,603 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLED in hbase:meta 2023-11-12T09:28:27,605 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:28:27,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=104, state=SUCCESS; CreateTableProcedure table=ns3:table3_restore in 1.1600 sec 2023-11-12T09:28:28,299 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:28:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=104 2023-11-12T09:28:28,557 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:28:28,557 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(233): Backup: backup_1699781286935 hdfs://localhost:34955/backupUT/backup_1699781286935/ns3/test-16997811824172/ 2023-11-12T09:28:28,557 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-12T09:28:28,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable ns2:table2 2023-11-12T09:28:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:table2 2023-11-12T09:28:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=107 2023-11-12T09:28:28,564 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781308563"}]},"ts":"1699781308563"} 2023-11-12T09:28:28,565 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2023-11-12T09:28:28,567 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set ns2:table2 to state=DISABLING 2023-11-12T09:28:28,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=3bddb82ee5b5de8b4c725633578df152, UNASSIGN}] 2023-11-12T09:28:28,570 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=3bddb82ee5b5de8b4c725633578df152, UNASSIGN 2023-11-12T09:28:28,570 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=3bddb82ee5b5de8b4c725633578df152, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:28,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; CloseRegionProcedure 3bddb82ee5b5de8b4c725633578df152, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:28:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=107 2023-11-12T09:28:28,723 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:28,724 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(119): Close 3bddb82ee5b5de8b4c725633578df152 2023-11-12T09:28:28,724 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1668): Closing 3bddb82ee5b5de8b4c725633578df152, disabling compactions & flushes 2023-11-12T09:28:28,724 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1690): Closing region ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152. 2023-11-12T09:28:28,724 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152. 2023-11-12T09:28:28,724 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152. after waiting 0 ms 2023-11-12T09:28:28,724 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152. 2023-11-12T09:28:28,731 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-12T09:28:28,732 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:28:28,733 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1909): Closed ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152. 2023-11-12T09:28:28,733 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1622): Region close journal for 3bddb82ee5b5de8b4c725633578df152: 2023-11-12T09:28:28,735 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(163): Closed 3bddb82ee5b5de8b4c725633578df152 2023-11-12T09:28:28,735 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=3bddb82ee5b5de8b4c725633578df152, regionState=CLOSED 2023-11-12T09:28:28,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=109, resume processing ppid=108 2023-11-12T09:28:28,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=109, ppid=108, state=SUCCESS; CloseRegionProcedure 3bddb82ee5b5de8b4c725633578df152, server=jenkins-hbase3.apache.org,36287,1699781172353 in 164 msec 2023-11-12T09:28:28,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=108, resume processing ppid=107 2023-11-12T09:28:28,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=108, ppid=107, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=3bddb82ee5b5de8b4c725633578df152, UNASSIGN in 170 msec 2023-11-12T09:28:28,740 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781308740"}]},"ts":"1699781308740"} 2023-11-12T09:28:28,742 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2023-11-12T09:28:28,744 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set ns2:table2 to state=DISABLED 2023-11-12T09:28:28,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=107, state=SUCCESS; DisableTableProcedure table=ns2:table2 in 185 msec 2023-11-12T09:28:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=107 2023-11-12T09:28:28,867 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$5(2449): Client=jenkins//172.31.12.81 delete ns2:table2 2023-11-12T09:28:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=110, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns2:table2 2023-11-12T09:28:28,870 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=110, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-12T09:28:28,870 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=110, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-12T09:28:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=110 2023-11-12T09:28:28,875 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152 2023-11-12T09:28:28,877 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152/f, FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152/recovered.edits] 2023-11-12T09:28:28,884 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152/f/20145e3258b94f36ab1dba1b24bff233_SeqId_4_ to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152/f/20145e3258b94f36ab1dba1b24bff233_SeqId_4_ 2023-11-12T09:28:28,888 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152/recovered.edits/6.seqid to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152/recovered.edits/6.seqid 2023-11-12T09:28:28,890 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/3bddb82ee5b5de8b4c725633578df152 2023-11-12T09:28:28,891 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived ns2:table2 regions 2023-11-12T09:28:28,894 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=110, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-12T09:28:28,900 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2023-11-12T09:28:28,902 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:table2' descriptor. 2023-11-12T09:28:28,904 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=110, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-12T09:28:28,904 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:table2' from region states. 2023-11-12T09:28:28,904 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781308904"}]},"ts":"9223372036854775807"} 2023-11-12T09:28:28,910 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:28:28,910 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 3bddb82ee5b5de8b4c725633578df152, NAME => 'ns2:table2,,1699781301620.3bddb82ee5b5de8b4c725633578df152.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:28:28,910 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:table2' as deleted. 2023-11-12T09:28:28,910 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781308910"}]},"ts":"9223372036854775807"} 2023-11-12T09:28:28,912 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:table2 state from META 2023-11-12T09:28:28,916 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=110, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-12T09:28:28,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=110, state=SUCCESS; DeleteTableProcedure table=ns2:table2 in 49 msec 2023-11-12T09:28:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=110 2023-11-12T09:28:28,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable ns3:table3_restore 2023-11-12T09:28:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:table3_restore 2023-11-12T09:28:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=111 2023-11-12T09:28:28,978 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781308978"}]},"ts":"1699781308978"} 2023-11-12T09:28:28,979 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLING in hbase:meta 2023-11-12T09:28:28,981 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set ns3:table3_restore to state=DISABLING 2023-11-12T09:28:28,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=19bae2349079b2c732d5ae3c1c5c9eda, UNASSIGN}] 2023-11-12T09:28:28,984 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=112, ppid=111, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=19bae2349079b2c732d5ae3c1c5c9eda, UNASSIGN 2023-11-12T09:28:28,984 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=112 updating hbase:meta row=19bae2349079b2c732d5ae3c1c5c9eda, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:28,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE; CloseRegionProcedure 19bae2349079b2c732d5ae3c1c5c9eda, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:28:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=111 2023-11-12T09:28:29,137 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:29,138 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] handler.UnassignRegionHandler(119): Close 19bae2349079b2c732d5ae3c1c5c9eda 2023-11-12T09:28:29,140 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1668): Closing 19bae2349079b2c732d5ae3c1c5c9eda, disabling compactions & flushes 2023-11-12T09:28:29,140 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda. 2023-11-12T09:28:29,140 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda. 2023-11-12T09:28:29,140 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda. after waiting 0 ms 2023-11-12T09:28:29,140 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda. 2023-11-12T09:28:29,144 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/19bae2349079b2c732d5ae3c1c5c9eda/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:28:29,145 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:28:29,146 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda. 2023-11-12T09:28:29,146 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1622): Region close journal for 19bae2349079b2c732d5ae3c1c5c9eda: 2023-11-12T09:28:29,148 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] handler.UnassignRegionHandler(163): Closed 19bae2349079b2c732d5ae3c1c5c9eda 2023-11-12T09:28:29,149 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=112 updating hbase:meta row=19bae2349079b2c732d5ae3c1c5c9eda, regionState=CLOSED 2023-11-12T09:28:29,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=113, resume processing ppid=112 2023-11-12T09:28:29,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=113, ppid=112, state=SUCCESS; CloseRegionProcedure 19bae2349079b2c732d5ae3c1c5c9eda, server=jenkins-hbase3.apache.org,36287,1699781172353 in 164 msec 2023-11-12T09:28:29,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=112, resume processing ppid=111 2023-11-12T09:28:29,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=112, ppid=111, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=19bae2349079b2c732d5ae3c1c5c9eda, UNASSIGN in 169 msec 2023-11-12T09:28:29,153 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781309153"}]},"ts":"1699781309153"} 2023-11-12T09:28:29,154 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLED in hbase:meta 2023-11-12T09:28:29,156 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set ns3:table3_restore to state=DISABLED 2023-11-12T09:28:29,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=111, state=SUCCESS; DisableTableProcedure table=ns3:table3_restore in 184 msec 2023-11-12T09:28:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=111 2023-11-12T09:28:29,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$5(2449): Client=jenkins//172.31.12.81 delete ns3:table3_restore 2023-11-12T09:28:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=114, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns3:table3_restore 2023-11-12T09:28:29,284 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=114, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-12T09:28:29,285 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=114, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-12T09:28:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=114 2023-11-12T09:28:29,288 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/19bae2349079b2c732d5ae3c1c5c9eda 2023-11-12T09:28:29,290 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/19bae2349079b2c732d5ae3c1c5c9eda/f, FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/19bae2349079b2c732d5ae3c1c5c9eda/recovered.edits] 2023-11-12T09:28:29,295 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/19bae2349079b2c732d5ae3c1c5c9eda/recovered.edits/4.seqid to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/ns3/table3_restore/19bae2349079b2c732d5ae3c1c5c9eda/recovered.edits/4.seqid 2023-11-12T09:28:29,296 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/19bae2349079b2c732d5ae3c1c5c9eda 2023-11-12T09:28:29,296 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(293): Archived ns3:table3_restore regions 2023-11-12T09:28:29,298 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=114, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-12T09:28:29,300 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:table3_restore from hbase:meta 2023-11-12T09:28:29,302 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:table3_restore' descriptor. 2023-11-12T09:28:29,303 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=114, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-12T09:28:29,303 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:table3_restore' from region states. 2023-11-12T09:28:29,303 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781309303"}]},"ts":"9223372036854775807"} 2023-11-12T09:28:29,304 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:28:29,305 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 19bae2349079b2c732d5ae3c1c5c9eda, NAME => 'ns3:table3_restore,,1699781306444.19bae2349079b2c732d5ae3c1c5c9eda.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:28:29,305 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:table3_restore' as deleted. 2023-11-12T09:28:29,305 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781309305"}]},"ts":"9223372036854775807"} 2023-11-12T09:28:29,306 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:table3_restore state from META 2023-11-12T09:28:29,309 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(127): Finished pid=114, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-12T09:28:29,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=114, state=SUCCESS; DeleteTableProcedure table=ns3:table3_restore in 28 msec 2023-11-12T09:28:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=114 2023-11-12T09:28:29,410 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultiple Thread=602 (was 598) - Thread LEAK? -, OpenFileDescriptor=704 (was 711), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=142 (was 122) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=6313 (was 5236) - AvailableMemoryMB LEAK? - 2023-11-12T09:28:29,410 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-11-12T09:28:29,426 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleOverwriteCommand Thread=602, OpenFileDescriptor=704, MaxFileDescriptor=60000, SystemLoadAverage=142, ProcessCount=171, AvailableMemoryMB=6309 2023-11-12T09:28:29,426 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-11-12T09:28:29,426 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(234): create full backup image on multiple tables: command-line 2023-11-12T09:28:29,433 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a47f94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:29,437 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:29,440 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57336, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:29,442 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:29,444 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:55496, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:28:29,455 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:28:29,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:28:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781309460 (current time:1699781309460). 2023-11-12T09:28:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:28:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:28:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c9d133, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:29,469 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57344, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:28:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=115, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:28:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 115 2023-11-12T09:28:29,476 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:28:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2023-11-12T09:28:29,477 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:28:29,485 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:28:29,490 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:28:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2023-11-12T09:28:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2023-11-12T09:28:29,897 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:28:29,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0}] 2023-11-12T09:28:29,902 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:28:30,053 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:30,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=116 2023-11-12T09:28:30,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:28:30,054 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-11-12T09:28:30,058 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/0703c67cb50e4c9eb5a47d31720493c9 is 171, key is trslm:hdfs://localhost:34955/backupUT\x00ns2:test-16997811824171/meta:log-roll-map/1699781300753/Put/seqid=0 2023-11-12T09:28:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2023-11-12T09:28:30,463 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/0703c67cb50e4c9eb5a47d31720493c9 2023-11-12T09:28:30,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/11d8abd27f484bf1b60368d701936880 is 524, key is session:backup_1699781286935/session:context/1699781301585/Put/seqid=0 2023-11-12T09:28:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2023-11-12T09:28:30,876 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/11d8abd27f484bf1b60368d701936880 2023-11-12T09:28:30,882 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/0703c67cb50e4c9eb5a47d31720493c9 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0703c67cb50e4c9eb5a47d31720493c9 2023-11-12T09:28:30,890 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0703c67cb50e4c9eb5a47d31720493c9, entries=6, sequenceid=94, filesize=5.8 K 2023-11-12T09:28:30,891 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/11d8abd27f484bf1b60368d701936880 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/11d8abd27f484bf1b60368d701936880 2023-11-12T09:28:30,903 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/11d8abd27f484bf1b60368d701936880, entries=2, sequenceid=94, filesize=5.5 K 2023-11-12T09:28:30,904 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 850ms, sequenceid=94, compaction requested=true 2023-11-12T09:28:30,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(2520): Flush status journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:28:30,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. for snapshot_backup_system completed. 2023-11-12T09:28:30,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:28:30,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:28:30,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0703c67cb50e4c9eb5a47d31720493c9] hfiles 2023-11-12T09:28:30,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db for snapshot=snapshot_backup_system 2023-11-12T09:28:30,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81 for snapshot=snapshot_backup_system 2023-11-12T09:28:30,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc for snapshot=snapshot_backup_system 2023-11-12T09:28:30,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd for snapshot=snapshot_backup_system 2023-11-12T09:28:30,904 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00 for snapshot=snapshot_backup_system 2023-11-12T09:28:30,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa for snapshot=snapshot_backup_system 2023-11-12T09:28:30,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87 for snapshot=snapshot_backup_system 2023-11-12T09:28:30,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0703c67cb50e4c9eb5a47d31720493c9 for snapshot=snapshot_backup_system 2023-11-12T09:28:30,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/11d8abd27f484bf1b60368d701936880] hfiles 2023-11-12T09:28:30,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (1/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 for snapshot=snapshot_backup_system 2023-11-12T09:28:30,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (2/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b for snapshot=snapshot_backup_system 2023-11-12T09:28:30,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (3/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd for snapshot=snapshot_backup_system 2023-11-12T09:28:30,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (4/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c for snapshot=snapshot_backup_system 2023-11-12T09:28:30,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (5/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5 for snapshot=snapshot_backup_system 2023-11-12T09:28:30,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (6/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8 for snapshot=snapshot_backup_system 2023-11-12T09:28:30,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (7/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3 for snapshot=snapshot_backup_system 2023-11-12T09:28:30,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (8/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244 for snapshot=snapshot_backup_system 2023-11-12T09:28:30,905 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (9/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/11d8abd27f484bf1b60368d701936880 for snapshot=snapshot_backup_system 2023-11-12T09:28:30,911 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:28:30,911 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=116 2023-11-12T09:28:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=116 2023-11-12T09:28:30,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 606338a328e82acae51355d485ed81f0 2023-11-12T09:28:30,913 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:28:30,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=116, resume processing ppid=115 2023-11-12T09:28:30,916 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:28:30,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=116, ppid=115, state=SUCCESS; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 in 1.0160 sec 2023-11-12T09:28:30,916 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:28:30,917 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:28:30,917 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:28:30,917 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:28:30,935 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:28:30,951 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:28:30,952 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:28:30,953 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:28:30,953 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 115 2023-11-12T09:28:30,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=115, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.4830 sec 2023-11-12T09:28:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2023-11-12T09:28:31,582 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(115): Backup backup_1699781309433 started at 1699781311582. 2023-11-12T09:28:31,583 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(119): Backup session backup_1699781309433 has been started. 2023-11-12T09:28:31,588 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:28:31,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:28:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:28:31,589 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:28:31,589 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:28:31,590 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:28:31,590 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:28:31,591 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:28:31,592 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:28:31,592 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:31,592 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:28:31,593 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:28:31,593 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:28:31,593 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:28:31,594 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:28:31,594 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:28:31,594 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:28:31,594 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:28:31,594 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:28:31,594 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:28:31,595 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:28:31,595 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:28:31,595 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:28:31,597 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:28:31,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:31,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:28:31,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:28:31,598 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:28:31,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:28:31,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:28:31,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:31,599 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:31,599 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:28:31,599 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:28:31,600 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2beed8c3[Count = 0] remaining members to acquire global barrier 2023-11-12T09:28:31,600 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:28:31,600 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:28:31,602 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:28:31,602 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:28:31,602 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:28:31,602 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:28:31,602 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:28:31,602 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:31,602 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781291112 highest: 1699781291112 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:31,602 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781291112) roll requested 2023-11-12T09:28:31,614 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781311602, exclude list is [], retry=0 2023-11-12T09:28:31,618 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:28:31,619 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781291112 with entries=15, filesize=4.08 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781311602 2023-11-12T09:28:31,619 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:28:31,619 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781291112 is not closed yet, will try archiving it next time 2023-11-12T09:28:31,620 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781276417 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781276417 2023-11-12T09:28:31,620 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781291131) roll requested 2023-11-12T09:28:31,624 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781291112 not finished, retry = 0 2023-11-12T09:28:31,634 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781311620.meta, exclude list is [], retry=0 2023-11-12T09:28:31,637 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:28:31,638 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781291131.meta with entries=24, filesize=5.75 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781311620.meta 2023-11-12T09:28:31,639 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:28:31,639 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781291131.meta is not closed yet, will try archiving it next time 2023-11-12T09:28:31,643 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-11-12T09:28:31,643 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781311602 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:31,651 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:28:31,652 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36287,1699781172353' in zk 2023-11-12T09:28:31,654 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:28:31,654 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:28:31,654 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:28:31,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:31,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:28:31,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:28:31,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:28:31,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:28:31,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:31,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:31,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:28:31,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:31,658 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:31,658 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36287,1699781172353': 2023-11-12T09:28:31,658 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:28:31,658 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:28:31,658 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:28:31,658 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:28:31,659 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:28:31,659 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:28:31,660 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:28:31,660 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:28:31,660 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:28:31,661 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:28:31,661 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:28:31,661 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:28:31,661 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:28:31,661 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:28:31,661 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:31,661 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:28:31,662 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:28:31,662 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:31,662 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:31,663 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:28:31,663 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:31,664 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:31,667 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:28:31,667 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:28:31,668 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:28:31,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:28:31,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:28:31,669 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:28:31,669 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:28:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:28:31,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } 2023-11-12T09:28:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781311773 (current time:1699781311773). 2023-11-12T09:28:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:28:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781311772_ns2_test-16997811824171 VERSION not specified, setting to 2 2023-11-12T09:28:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b7917f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:31,783 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57358, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:28:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=117, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } 2023-11-12T09:28:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 }, snapshot procedure id = 117 2023-11-12T09:28:31,787 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:28:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2023-11-12T09:28:31,788 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:28:31,790 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781311772_ns2_test-16997811824171 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:28:31,794 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:28:31,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2023-11-12T09:28:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2023-11-12T09:28:32,201 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:28:32,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95}] 2023-11-12T09:28:32,203 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:28:32,354 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:32,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=118 2023-11-12T09:28:32,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:28:32,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.HRegion(2520): Flush status journal for 6e68e35ec82e3efaab2edd4a90156b95: 2023-11-12T09:28:32,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. for snapshot_1699781311772_ns2_test-16997811824171 completed. 2023-11-12T09:28:32,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95.' region-info for snapshot=snapshot_1699781311772_ns2_test-16997811824171 2023-11-12T09:28:32,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:28:32,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_] hfiles 2023-11-12T09:28:32,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ for snapshot=snapshot_1699781311772_ns2_test-16997811824171 2023-11-12T09:28:32,361 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:28:32,361 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=118 2023-11-12T09:28:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=118 2023-11-12T09:28:32,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781311772_ns2_test-16997811824171 on region 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:28:32,363 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:28:32,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=118, resume processing ppid=117 2023-11-12T09:28:32,366 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:28:32,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=118, ppid=117, state=SUCCESS; SnapshotRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95 in 163 msec 2023-11-12T09:28:32,367 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:28:32,367 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:28:32,367 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781311772_ns2_test-16997811824171 2023-11-12T09:28:32,368 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781311772_ns2_test-16997811824171 2023-11-12T09:28:32,379 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:28:32,385 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:28:32,385 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781311772_ns2_test-16997811824171 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781311772_ns2_test-16997811824171 2023-11-12T09:28:32,386 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:28:32,386 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 }, snapshot procedure id = 117 2023-11-12T09:28:32,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=117, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1699781311772_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } in 602 msec 2023-11-12T09:28:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2023-11-12T09:28:32,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } 2023-11-12T09:28:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781312391 (current time:1699781312391). 2023-11-12T09:28:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:28:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781312391_ns3_test-16997811824172 VERSION not specified, setting to 2 2023-11-12T09:28:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a65fbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:32,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:32,403 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57370, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:28:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=119, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } 2023-11-12T09:28:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 }, snapshot procedure id = 119 2023-11-12T09:28:32,409 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:28:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2023-11-12T09:28:32,410 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:28:32,413 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781312391_ns3_test-16997811824172 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:28:32,417 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:28:32,433 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:28:32,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure 8e08148d1dda03a07c8babda3e33ae17}] 2023-11-12T09:28:32,435 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:28:32,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2023-11-12T09:28:32,587 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:32,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=120 2023-11-12T09:28:32,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:28:32,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.HRegion(2520): Flush status journal for 8e08148d1dda03a07c8babda3e33ae17: 2023-11-12T09:28:32,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. for snapshot_1699781312391_ns3_test-16997811824172 completed. 2023-11-12T09:28:32,588 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17.' region-info for snapshot=snapshot_1699781312391_ns3_test-16997811824172 2023-11-12T09:28:32,588 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:28:32,588 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-12T09:28:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2023-11-12T09:28:32,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:28:32,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=120 2023-11-12T09:28:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=120 2023-11-12T09:28:32,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781312391_ns3_test-16997811824172 on region 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:28:32,996 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:28:32,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=120, resume processing ppid=119 2023-11-12T09:28:32,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=120, ppid=119, state=SUCCESS; SnapshotRegionProcedure 8e08148d1dda03a07c8babda3e33ae17 in 563 msec 2023-11-12T09:28:32,997 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:28:32,998 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:28:32,998 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:28:32,998 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781312391_ns3_test-16997811824172 2023-11-12T09:28:32,999 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781312391_ns3_test-16997811824172 2023-11-12T09:28:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2023-11-12T09:28:33,412 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:28:33,416 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:28:33,417 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781312391_ns3_test-16997811824172 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781312391_ns3_test-16997811824172 2023-11-12T09:28:33,418 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:28:33,418 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 }, snapshot procedure id = 119 2023-11-12T09:28:33,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1699781312391_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } in 1.0130 sec 2023-11-12T09:28:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2023-11-12T09:28:33,515 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781309433 2023-11-12T09:28:33,515 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:28:33,515 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-12T09:28:33,515 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781309433_ns3:test-16997811824172 2023-11-12T09:28:33,515 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781312391_ns3_test-16997811824172 to hdfs://localhost:34955/backupUT/backup_1699781309433/ns3/test-16997811824172/ 2023-11-12T09:28:33,515 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:28:33,539 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:28:33,539 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_-1888331272_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781309433/ns3/test-16997811824172, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781309433/ns3/test-16997811824172/.hbase-snapshot/.tmp/snapshot_1699781312391_ns3_test-16997811824172 2023-11-12T09:28:33,539 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781312391_ns3_test-16997811824172. 2023-11-12T09:28:33,544 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781312391_ns3_test-16997811824172 to hdfs://localhost:34955/backupUT/backup_1699781309433/ns3/test-16997811824172/.hbase-snapshot/.tmp/snapshot_1699781312391_ns3_test-16997811824172 2023-11-12T09:28:33,965 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:33,966 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:33,966 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:33,967 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:34,235 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:28:34,968 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-2482880877093041307.jar 2023-11-12T09:28:34,969 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:34,969 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:34,970 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:34,970 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:34,970 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:34,971 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:34,971 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:34,971 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:34,971 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:34,972 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:28:34,973 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:28:34,973 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:28:34,973 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:28:34,974 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:28:34,974 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:28:34,975 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:28:34,975 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:28:34,975 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:28:34,976 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:28:34,976 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:28:34,977 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:28:34,977 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:34,978 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:34,978 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:34,979 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:34,979 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:34,979 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:34,980 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:35,026 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781312391_ns3_test-16997811824172' hfile list 2023-11-12T09:28:35,116 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781315073/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:28:35,116 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781315073/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:28:36,147 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:28:36,149 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:28:36,154 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781312391_ns3_test-16997811824172 2023-11-12T09:28:36,154 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781312391_ns3_test-16997811824172 finished. 2023-11-12T09:28:36,154 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781309433_ns2:test-16997811824171 2023-11-12T09:28:36,154 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781311772_ns2_test-16997811824171 to hdfs://localhost:34955/backupUT/backup_1699781309433/ns2/test-16997811824171/ 2023-11-12T09:28:36,154 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:28:36,177 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:28:36,177 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_1026632123_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781309433/ns2/test-16997811824171, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781309433/ns2/test-16997811824171/.hbase-snapshot/.tmp/snapshot_1699781311772_ns2_test-16997811824171 2023-11-12T09:28:36,177 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781311772_ns2_test-16997811824171. 2023-11-12T09:28:36,186 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781311772_ns2_test-16997811824171 to hdfs://localhost:34955/backupUT/backup_1699781309433/ns2/test-16997811824171/.hbase-snapshot/.tmp/snapshot_1699781311772_ns2_test-16997811824171 2023-11-12T09:28:36,606 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:36,606 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:36,607 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:36,607 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:37,606 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-1255442657877851305.jar 2023-11-12T09:28:37,606 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:37,606 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:37,607 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:37,607 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:37,608 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:37,608 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:37,608 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:37,609 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:37,609 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:37,610 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:28:37,610 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:28:37,611 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:28:37,611 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:28:37,611 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:28:37,612 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:28:37,612 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:28:37,612 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:28:37,613 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:28:37,613 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:28:37,614 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:28:37,614 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:28:37,614 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:37,615 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:37,616 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:37,616 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:37,616 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:37,617 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:37,617 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:37,659 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781311772_ns2_test-16997811824171' hfile list 2023-11-12T09:28:37,661 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-12T09:28:37,730 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781317696/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:28:37,730 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781317696/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:28:37,789 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:28:37,813 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ output=hdfs://localhost:34955/backupUT/backup_1699781309433/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ 2023-11-12T09:28:37,813 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 4.160M/sec 2023-11-12T09:28:38,767 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:28:38,768 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:28:38,775 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781311772_ns2_test-16997811824171 2023-11-12T09:28:38,775 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781311772_ns2_test-16997811824171 finished. 2023-11-12T09:28:38,775 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): ns3:test-16997811824172 2023-11-12T09:28:38,775 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): ns2:test-16997811824171 2023-11-12T09:28:38,782 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781309433 2023-11-12T09:28:38,782 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:28:39,191 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781309433/.backup.manifest 2023-11-12T09:28:39,191 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781309433 2023-11-12T09:28:39,191 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:28:39,596 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781309433/.backup.manifest 2023-11-12T09:28:39,596 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(358): Backup backup_1699781309433 finished: type=FULL,tablelist=ns3:test-16997811824172;ns2:test-16997811824171,targetRootDir=hdfs://localhost:34955/backupUT,startts=1699781311582,completets=1699781318782,bytescopied=0 2023-11-12T09:28:39,596 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:28:39,597 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781312391_ns3_test-16997811824172 2023-11-12T09:28:39,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781312391_ns3_test-16997811824172" 2023-11-12T09:28:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781312391_ns3_test-16997811824172 2023-11-12T09:28:39,601 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781312391_ns3_test-16997811824172 for backup backup_1699781309433 succeeded. 2023-11-12T09:28:39,601 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781311772_ns2_test-16997811824171 2023-11-12T09:28:39,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781311772_ns2_test-16997811824171" 2023-11-12T09:28:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781311772_ns2_test-16997811824171 2023-11-12T09:28:39,604 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781311772_ns2_test-16997811824171 for backup backup_1699781309433 succeeded. 2023-11-12T09:28:39,606 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:28:39,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:28:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:28:39,612 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:28:39,614 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:28:39,619 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(376): Backup backup_1699781309433 completed. 2023-11-12T09:28:39,619 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:39,631 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] backup.RestoreDriver(101): Found -overwrite option in restore command, will overwrite to existing table if any in the restore target 2023-11-12T09:28:39,643 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4183f85c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:39,649 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:39,651 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35734, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:39,655 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781309433/.backup.manifest 2023-11-12T09:28:39,658 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781309433/.backup.manifest 2023-11-12T09:28:39,658 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781309433/.backup.manifest 2023-11-12T09:28:39,661 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781309433/.backup.manifest 2023-11-12T09:28:39,661 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781309433/.backup.manifest 2023-11-12T09:28:39,663 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781309433/.backup.manifest 2023-11-12T09:28:39,663 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16997811824171' to 'ns2:test-16997811824171' from full backup image hdfs://localhost:34955/backupUT/backup_1699781309433/ns2/test-16997811824171 2023-11-12T09:28:39,676 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(476): Truncating exising target table 'ns2:test-16997811824171', preserving region splits 2023-11-12T09:28:39,677 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:39,679 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39822, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:28:39,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable ns2:test-16997811824171 2023-11-12T09:28:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=121, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:test-16997811824171 2023-11-12T09:28:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=121 2023-11-12T09:28:39,684 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997811824171","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781319684"}]},"ts":"1699781319684"} 2023-11-12T09:28:39,685 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997811824171, state=DISABLING in hbase:meta 2023-11-12T09:28:39,687 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set ns2:test-16997811824171 to state=DISABLING 2023-11-12T09:28:39,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16997811824171, region=6e68e35ec82e3efaab2edd4a90156b95, UNASSIGN}] 2023-11-12T09:28:39,690 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16997811824171, region=6e68e35ec82e3efaab2edd4a90156b95, UNASSIGN 2023-11-12T09:28:39,691 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=6e68e35ec82e3efaab2edd4a90156b95, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:39,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE; CloseRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:28:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=121 2023-11-12T09:28:39,843 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:39,844 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(119): Close 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:28:39,844 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1668): Closing 6e68e35ec82e3efaab2edd4a90156b95, disabling compactions & flushes 2023-11-12T09:28:39,844 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1690): Closing region ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:28:39,844 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:28:39,844 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. after waiting 0 ms 2023-11-12T09:28:39,844 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:28:39,851 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-11-12T09:28:39,852 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:28:39,853 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1909): Closed ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95. 2023-11-12T09:28:39,853 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1622): Region close journal for 6e68e35ec82e3efaab2edd4a90156b95: 2023-11-12T09:28:39,854 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(163): Closed 6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:28:39,854 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=6e68e35ec82e3efaab2edd4a90156b95, regionState=CLOSED 2023-11-12T09:28:39,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=123, resume processing ppid=122 2023-11-12T09:28:39,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=123, ppid=122, state=SUCCESS; CloseRegionProcedure 6e68e35ec82e3efaab2edd4a90156b95, server=jenkins-hbase3.apache.org,36287,1699781172353 in 164 msec 2023-11-12T09:28:39,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=122, resume processing ppid=121 2023-11-12T09:28:39,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=122, ppid=121, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16997811824171, region=6e68e35ec82e3efaab2edd4a90156b95, UNASSIGN in 170 msec 2023-11-12T09:28:39,858 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997811824171","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781319858"}]},"ts":"1699781319858"} 2023-11-12T09:28:39,859 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997811824171, state=DISABLED in hbase:meta 2023-11-12T09:28:39,862 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set ns2:test-16997811824171 to state=DISABLED 2023-11-12T09:28:39,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=121, state=SUCCESS; DisableTableProcedure table=ns2:test-16997811824171 in 182 msec 2023-11-12T09:28:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=121 2023-11-12T09:28:39,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$6(2481): Client=jenkins//172.31.12.81 truncate ns2:test-16997811824171 2023-11-12T09:28:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=124, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns2:test-16997811824171 preserveSplits=true) 2023-11-12T09:28:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=124 2023-11-12T09:28:39,989 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:test-16997811824171' regions in transition 2023-11-12T09:28:39,992 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:28:39,993 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f, FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/recovered.edits] 2023-11-12T09:28:39,999 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ 2023-11-12T09:28:40,002 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/recovered.edits/9.seqid to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/recovered.edits/9.seqid 2023-11-12T09:28:40,006 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:28:40,006 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived ns2:test-16997811824171 regions 2023-11-12T09:28:40,010 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:test-16997811824171 from hbase:meta 2023-11-12T09:28:40,012 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:test-16997811824171' descriptor. 2023-11-12T09:28:40,012 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:test-16997811824171' from region states. 2023-11-12T09:28:40,013 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781320012"}]},"ts":"9223372036854775807"} 2023-11-12T09:28:40,014 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:28:40,014 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 6e68e35ec82e3efaab2edd4a90156b95, NAME => 'ns2:test-16997811824171,,1699781206106.6e68e35ec82e3efaab2edd4a90156b95.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:28:40,014 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:test-16997811824171' as deleted. 2023-11-12T09:28:40,014 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16997811824171","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781320014"}]},"ts":"9223372036854775807"} 2023-11-12T09:28:40,015 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:test-16997811824171 state from META 2023-11-12T09:28:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=124 2023-11-12T09:28:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=124 2023-11-12T09:28:40,347 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997811824171 2023-11-12T09:28:40,426 INFO [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d347587386821efa96e22f590d908cde, NAME => 'ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16997811824171', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:28:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=124 2023-11-12T09:28:40,832 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:40,832 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1668): Closing d347587386821efa96e22f590d908cde, disabling compactions & flushes 2023-11-12T09:28:40,832 INFO [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:28:40,832 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:28:40,832 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. after waiting 0 ms 2023-11-12T09:28:40,832 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:28:40,832 INFO [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:28:40,832 DEBUG [RegionOpenAndInit-ns2:test-16997811824171-pool-0 {}] regionserver.HRegion(1622): Region close journal for d347587386821efa96e22f590d908cde: 2023-11-12T09:28:40,833 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781320833"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781320833"}]},"ts":"1699781320833"} 2023-11-12T09:28:40,834 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:28:40,835 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997811824171","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781320835"}]},"ts":"1699781320835"} 2023-11-12T09:28:40,836 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997811824171, state=ENABLING in hbase:meta 2023-11-12T09:28:40,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997811824171, region=d347587386821efa96e22f590d908cde, ASSIGN}] 2023-11-12T09:28:40,842 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997811824171, region=d347587386821efa96e22f590d908cde, ASSIGN 2023-11-12T09:28:40,842 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16997811824171, region=d347587386821efa96e22f590d908cde, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:28:40,994 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=125 updating hbase:meta row=d347587386821efa96e22f590d908cde, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:40,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE; OpenRegionProcedure d347587386821efa96e22f590d908cde, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:28:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=124 2023-11-12T09:28:41,147 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:41,150 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(130): Open ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:28:41,150 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7254): Opening region: {ENCODED => d347587386821efa96e22f590d908cde, NAME => 'ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:41,151 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:28:41,151 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997811824171 d347587386821efa96e22f590d908cde 2023-11-12T09:28:41,151 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(888): Instantiated ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:41,151 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7296): checking encryption for d347587386821efa96e22f590d908cde 2023-11-12T09:28:41,151 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7299): checking classloading for d347587386821efa96e22f590d908cde 2023-11-12T09:28:41,153 INFO [StoreOpener-d347587386821efa96e22f590d908cde-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d347587386821efa96e22f590d908cde 2023-11-12T09:28:41,154 INFO [StoreOpener-d347587386821efa96e22f590d908cde-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d347587386821efa96e22f590d908cde columnFamilyName f 2023-11-12T09:28:41,154 DEBUG [StoreOpener-d347587386821efa96e22f590d908cde-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:41,155 INFO [StoreOpener-d347587386821efa96e22f590d908cde-1 {}] regionserver.HStore(324): Store=d347587386821efa96e22f590d908cde/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:41,155 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde 2023-11-12T09:28:41,156 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde 2023-11-12T09:28:41,158 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1076): writing seq id for d347587386821efa96e22f590d908cde 2023-11-12T09:28:41,160 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:28:41,160 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): Opened d347587386821efa96e22f590d908cde; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11737556160, jitterRate=0.09314510226249695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:28:41,161 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(992): Region open journal for d347587386821efa96e22f590d908cde: 2023-11-12T09:28:41,162 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde., pid=126, masterSystemTime=1699781321147 2023-11-12T09:28:41,163 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:28:41,163 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(158): Opened ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:28:41,163 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=125 updating hbase:meta row=d347587386821efa96e22f590d908cde, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:41,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=126, resume processing ppid=125 2023-11-12T09:28:41,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=126, ppid=125, state=SUCCESS; OpenRegionProcedure d347587386821efa96e22f590d908cde, server=jenkins-hbase3.apache.org,36287,1699781172353 in 170 msec 2023-11-12T09:28:41,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=125, resume processing ppid=124 2023-11-12T09:28:41,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=125, ppid=124, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16997811824171, region=d347587386821efa96e22f590d908cde, ASSIGN in 326 msec 2023-11-12T09:28:41,168 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997811824171","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781321168"}]},"ts":"1699781321168"} 2023-11-12T09:28:41,169 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997811824171, state=ENABLED in hbase:meta 2023-11-12T09:28:41,172 DEBUG [PEWorker-4 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:test-16997811824171' completed 2023-11-12T09:28:41,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=124, state=SUCCESS; TruncateTableProcedure (table=ns2:test-16997811824171 preserveSplits=true) in 1.1860 sec 2023-11-12T09:28:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=124 2023-11-12T09:28:42,099 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34955/backupUT/backup_1699781309433/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95 from hbase tables ns2:test-16997811824171 to tables ns2:test-16997811824171 2023-11-12T09:28:42,099 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16997811824171 into ns2:test-16997811824171 2023-11-12T09:28:42,101 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781322100 from hdfs://localhost:34955/backupUT/backup_1699781309433/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95 2023-11-12T09:28:42,106 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad15f95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:42,107 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:42,108 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39836, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:28:42,109 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:28:42,109 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16997811824171 2023-11-12T09:28:42,112 DEBUG [hconnection-0x2742161-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:42,113 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35738, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:42,116 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16997811824171: 2023-11-12T09:28:42,116 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-12T09:28:42,116 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/partitions_c3ebded9-d064-48b9-9a3c-ab620b93658d 2023-11-12T09:28:42,524 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:42,525 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:42,526 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:42,526 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,547 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-2982694708326842641.jar 2023-11-12T09:28:43,548 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,548 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,548 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,549 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,549 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,550 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,550 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,550 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,550 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,551 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:28:43,552 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:28:43,552 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:28:43,553 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:28:43,553 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:28:43,553 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:28:43,554 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:28:43,554 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:28:43,554 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:28:43,555 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:28:43,555 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:28:43,556 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:28:43,556 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,556 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,557 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,557 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,557 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,558 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:43,558 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:43,558 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16997811824171 2023-11-12T09:28:43,559 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:43,559 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-12T09:28:43,559 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-12T09:28:43,559 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:28:43,571 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-12T09:28:43,648 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781323612/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:28:43,648 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781323612/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:28:43,697 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34955/backupUT/backup_1699781309433/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/6e68e35ec82e3efaab2edd4a90156b95/f/41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_ 2023-11-12T09:28:43,701 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-12T09:28:43,724 DEBUG [pool-441-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f443a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:43,725 DEBUG [hconnection-0x49c5ab53-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:43,727 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35752, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:43,730 DEBUG [pool-441-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:43,730 DEBUG [pool-441-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-12T09:28:43,730 DEBUG [pool-441-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:28:43,737 DEBUG [pool-441-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781322100/_temporary/0/_temporary/attempt_local311708625_0019_r_000000_0/f/00156ef48f32441099b9e953c3def95f is 37, key is row10/f:q1/1699781187295/Put/seqid=0 2023-11-12T09:28:43,739 WARN [Thread-1917 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36287] Specified favored nodes: [jenkins-hbase3.apache.org:36287] 2023-11-12T09:28:44,692 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781322100 2023-11-12T09:28:44,698 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@419fe5e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:44,699 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:44,701 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35756, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:44,706 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781322100/_SUCCESS 2023-11-12T09:28:44,710 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:44,711 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39846, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:28:44,719 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781322100/f/00156ef48f32441099b9e953c3def95f first=Optional[row0] last=Optional[row98] 2023-11-12T09:28:44,720 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde., hostname=jenkins-hbase3.apache.org,36287,1699781172353, seqNum=2 for row with hfile group [{f,hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781322100/f/00156ef48f32441099b9e953c3def95f}] 2023-11-12T09:28:44,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781322100/f/00156ef48f32441099b9e953c3def95f for inclusion in d347587386821efa96e22f590d908cde/f 2023-11-12T09:28:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-12T09:28:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-12T09:28:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegion(2520): Flush status journal for d347587386821efa96e22f590d908cde: 2023-11-12T09:28:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-test-16997811824171-1699781322100/f/00156ef48f32441099b9e953c3def95f to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_test-16997811824171__fmdt8tedcdul82oqd15mpeq5kq08dhlbncnpbfi3ned55v6ailfv0n5rpshu75hr/f/00156ef48f32441099b9e953c3def95f 2023-11-12T09:28:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_test-16997811824171__fmdt8tedcdul82oqd15mpeq5kq08dhlbncnpbfi3ned55v6ailfv0n5rpshu75hr/f/00156ef48f32441099b9e953c3def95f as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_ 2023-11-12T09:28:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f38ece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:44,742 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39862, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:28:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:44,747 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35768, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(428): write bulk load descriptor to backup ns2:test-16997811824171 with 1 entries 2023-11-12T09:28:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(1602): writing raw bulk path hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_ for ns2:test-16997811824171 d347587386821efa96e22f590d908cde 2023-11-12T09:28:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(435): written 1 rows for bulk load of ns2:test-16997811824171 2023-11-12T09:28:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:44,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_test-16997811824171__fmdt8tedcdul82oqd15mpeq5kq08dhlbncnpbfi3ned55v6ailfv0n5rpshu75hr/f/00156ef48f32441099b9e953c3def95f into d347587386821efa96e22f590d908cde/f as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_ - updating store file list. 2023-11-12T09:28:44,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_ into d347587386821efa96e22f590d908cde/f 2023-11-12T09:28:44,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_test-16997811824171__fmdt8tedcdul82oqd15mpeq5kq08dhlbncnpbfi3ned55v6ailfv0n5rpshu75hr/f/00156ef48f32441099b9e953c3def95f into d347587386821efa96e22f590d908cde/f (new location: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_) 2023-11-12T09:28:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_test-16997811824171__fmdt8tedcdul82oqd15mpeq5kq08dhlbncnpbfi3ned55v6ailfv0n5rpshu75hr/f/00156ef48f32441099b9e953c3def95f 2023-11-12T09:28:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b9cc1d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:44,777 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39876, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:28:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:44,783 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:35780, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-12T09:28:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup ns2:test-16997811824171 with 1 entries 2023-11-12T09:28:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(1523): writing done bulk path hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_ for ns2:test-16997811824171 d347587386821efa96e22f590d908cde 2023-11-12T09:28:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] impl.BackupSystemTable(414): written 1 rows for bulk load of ns2:test-16997811824171 2023-11-12T09:28:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:44,801 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:44,801 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-12T09:28:44,801 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:28:44,801 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(233): Backup: backup_1699781309433 hdfs://localhost:34955/backupUT/backup_1699781309433/ns2/test-16997811824171/ 2023-11-12T09:28:44,802 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781309433/.backup.manifest 2023-11-12T09:28:44,805 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781309433/.backup.manifest 2023-11-12T09:28:44,805 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16997811824172' to 'ns3:test-16997811824172' from full backup image hdfs://localhost:34955/backupUT/backup_1699781309433/ns3/test-16997811824172 2023-11-12T09:28:44,811 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:34955/backupUT/backup_1699781309433/ns3/test-16997811824172/archive/data/ns3/test-16997811824172 does not exists 2023-11-12T09:28:44,811 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16997811824172, will only create table 2023-11-12T09:28:44,811 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(476): Truncating exising target table 'ns3:test-16997811824172', preserving region splits 2023-11-12T09:28:44,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable ns3:test-16997811824172 2023-11-12T09:28:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:test-16997811824172 2023-11-12T09:28:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=127 2023-11-12T09:28:44,815 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997811824172","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781324815"}]},"ts":"1699781324815"} 2023-11-12T09:28:44,816 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997811824172, state=DISABLING in hbase:meta 2023-11-12T09:28:44,818 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set ns3:test-16997811824172 to state=DISABLING 2023-11-12T09:28:44,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16997811824172, region=8e08148d1dda03a07c8babda3e33ae17, UNASSIGN}] 2023-11-12T09:28:44,820 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16997811824172, region=8e08148d1dda03a07c8babda3e33ae17, UNASSIGN 2023-11-12T09:28:44,820 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=8e08148d1dda03a07c8babda3e33ae17, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:44,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE; CloseRegionProcedure 8e08148d1dda03a07c8babda3e33ae17, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:28:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=127 2023-11-12T09:28:44,973 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:44,973 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(119): Close 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:28:44,973 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1668): Closing 8e08148d1dda03a07c8babda3e33ae17, disabling compactions & flushes 2023-11-12T09:28:44,973 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1690): Closing region ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:28:44,973 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:28:44,973 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. after waiting 0 ms 2023-11-12T09:28:44,973 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:28:44,977 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/8e08148d1dda03a07c8babda3e33ae17/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=1 2023-11-12T09:28:44,978 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:28:44,979 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1909): Closed ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17. 2023-11-12T09:28:44,979 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1622): Region close journal for 8e08148d1dda03a07c8babda3e33ae17: 2023-11-12T09:28:44,981 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(163): Closed 8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:28:44,981 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=8e08148d1dda03a07c8babda3e33ae17, regionState=CLOSED 2023-11-12T09:28:44,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=129, resume processing ppid=128 2023-11-12T09:28:44,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=129, ppid=128, state=SUCCESS; CloseRegionProcedure 8e08148d1dda03a07c8babda3e33ae17, server=jenkins-hbase3.apache.org,36287,1699781172353 in 161 msec 2023-11-12T09:28:44,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=128, resume processing ppid=127 2023-11-12T09:28:44,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=128, ppid=127, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16997811824172, region=8e08148d1dda03a07c8babda3e33ae17, UNASSIGN in 166 msec 2023-11-12T09:28:44,986 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997811824172","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781324985"}]},"ts":"1699781324985"} 2023-11-12T09:28:44,986 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997811824172, state=DISABLED in hbase:meta 2023-11-12T09:28:44,989 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set ns3:test-16997811824172 to state=DISABLED 2023-11-12T09:28:44,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=127, state=SUCCESS; DisableTableProcedure table=ns3:test-16997811824172 in 177 msec 2023-11-12T09:28:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=127 2023-11-12T09:28:45,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$6(2481): Client=jenkins//172.31.12.81 truncate ns3:test-16997811824172 2023-11-12T09:28:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=130, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns3:test-16997811824172 preserveSplits=true) 2023-11-12T09:28:45,121 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(87): waiting for 'ns3:test-16997811824172' regions in transition 2023-11-12T09:28:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=130 2023-11-12T09:28:45,124 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:28:45,126 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/8e08148d1dda03a07c8babda3e33ae17/f, FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/8e08148d1dda03a07c8babda3e33ae17/recovered.edits] 2023-11-12T09:28:45,132 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/8e08148d1dda03a07c8babda3e33ae17/recovered.edits/7.seqid to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/ns3/test-16997811824172/8e08148d1dda03a07c8babda3e33ae17/recovered.edits/7.seqid 2023-11-12T09:28:45,132 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/8e08148d1dda03a07c8babda3e33ae17 2023-11-12T09:28:45,132 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived ns3:test-16997811824172 regions 2023-11-12T09:28:45,137 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:test-16997811824172 from hbase:meta 2023-11-12T09:28:45,138 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:test-16997811824172' descriptor. 2023-11-12T09:28:45,139 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:test-16997811824172' from region states. 2023-11-12T09:28:45,139 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781325139"}]},"ts":"9223372036854775807"} 2023-11-12T09:28:45,141 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:28:45,141 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 8e08148d1dda03a07c8babda3e33ae17, NAME => 'ns3:test-16997811824172,,1699781211545.8e08148d1dda03a07c8babda3e33ae17.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:28:45,141 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:test-16997811824172' as deleted. 2023-11-12T09:28:45,141 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16997811824172","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781325141"}]},"ts":"9223372036854775807"} 2023-11-12T09:28:45,143 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:test-16997811824172 state from META 2023-11-12T09:28:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=130 2023-11-12T09:28:45,410 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:28:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=130 2023-11-12T09:28:45,552 INFO [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => f50c319a7db3e25c53784414ac224095, NAME => 'ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16997811824172', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:28:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=130 2023-11-12T09:28:45,959 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:45,959 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1668): Closing f50c319a7db3e25c53784414ac224095, disabling compactions & flushes 2023-11-12T09:28:45,959 INFO [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:28:45,959 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:28:45,959 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. after waiting 0 ms 2023-11-12T09:28:45,960 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:28:45,960 INFO [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:28:45,960 DEBUG [RegionOpenAndInit-ns3:test-16997811824172-pool-0 {}] regionserver.HRegion(1622): Region close journal for f50c319a7db3e25c53784414ac224095: 2023-11-12T09:28:45,961 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781325961"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781325961"}]},"ts":"1699781325961"} 2023-11-12T09:28:45,963 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:28:45,964 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997811824172","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781325963"}]},"ts":"1699781325963"} 2023-11-12T09:28:45,965 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997811824172, state=ENABLING in hbase:meta 2023-11-12T09:28:45,970 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997811824172, region=f50c319a7db3e25c53784414ac224095, ASSIGN}] 2023-11-12T09:28:45,971 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997811824172, region=f50c319a7db3e25c53784414ac224095, ASSIGN 2023-11-12T09:28:45,972 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16997811824172, region=f50c319a7db3e25c53784414ac224095, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:28:46,124 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=131 updating hbase:meta row=f50c319a7db3e25c53784414ac224095, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:46,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE; OpenRegionProcedure f50c319a7db3e25c53784414ac224095, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:28:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=130 2023-11-12T09:28:46,278 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:46,282 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(130): Open ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:28:46,282 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7254): Opening region: {ENCODED => f50c319a7db3e25c53784414ac224095, NAME => 'ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:28:46,282 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:28:46,283 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997811824172 f50c319a7db3e25c53784414ac224095 2023-11-12T09:28:46,283 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(888): Instantiated ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:28:46,283 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7296): checking encryption for f50c319a7db3e25c53784414ac224095 2023-11-12T09:28:46,283 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7299): checking classloading for f50c319a7db3e25c53784414ac224095 2023-11-12T09:28:46,284 INFO [StoreOpener-f50c319a7db3e25c53784414ac224095-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f50c319a7db3e25c53784414ac224095 2023-11-12T09:28:46,286 INFO [StoreOpener-f50c319a7db3e25c53784414ac224095-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f50c319a7db3e25c53784414ac224095 columnFamilyName f 2023-11-12T09:28:46,286 DEBUG [StoreOpener-f50c319a7db3e25c53784414ac224095-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:28:46,287 INFO [StoreOpener-f50c319a7db3e25c53784414ac224095-1 {}] regionserver.HStore(324): Store=f50c319a7db3e25c53784414ac224095/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:28:46,288 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/f50c319a7db3e25c53784414ac224095 2023-11-12T09:28:46,288 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/f50c319a7db3e25c53784414ac224095 2023-11-12T09:28:46,290 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1076): writing seq id for f50c319a7db3e25c53784414ac224095 2023-11-12T09:28:46,293 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/f50c319a7db3e25c53784414ac224095/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:28:46,293 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): Opened f50c319a7db3e25c53784414ac224095; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9848885920, jitterRate=-0.08275102078914642}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:28:46,294 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(992): Region open journal for f50c319a7db3e25c53784414ac224095: 2023-11-12T09:28:46,295 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095., pid=132, masterSystemTime=1699781326278 2023-11-12T09:28:46,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:28:46,297 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(158): Opened ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:28:46,298 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=131 updating hbase:meta row=f50c319a7db3e25c53784414ac224095, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:46,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=132, resume processing ppid=131 2023-11-12T09:28:46,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=132, ppid=131, state=SUCCESS; OpenRegionProcedure f50c319a7db3e25c53784414ac224095, server=jenkins-hbase3.apache.org,36287,1699781172353 in 173 msec 2023-11-12T09:28:46,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=131, resume processing ppid=130 2023-11-12T09:28:46,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=131, ppid=130, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16997811824172, region=f50c319a7db3e25c53784414ac224095, ASSIGN in 331 msec 2023-11-12T09:28:46,303 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997811824172","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781326303"}]},"ts":"1699781326303"} 2023-11-12T09:28:46,304 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997811824172, state=ENABLED in hbase:meta 2023-11-12T09:28:46,306 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(145): truncate 'ns3:test-16997811824172' completed 2023-11-12T09:28:46,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=130, state=SUCCESS; TruncateTableProcedure (table=ns3:test-16997811824172 preserveSplits=true) in 1.1880 sec 2023-11-12T09:28:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=130 2023-11-12T09:28:47,230 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:28:47,230 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(233): Backup: backup_1699781309433 hdfs://localhost:34955/backupUT/backup_1699781309433/ns3/test-16997811824172/ 2023-11-12T09:28:47,230 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-12T09:28:47,231 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:47,250 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleOverwriteCommand Thread=607 (was 602) - Thread LEAK? -, OpenFileDescriptor=709 (was 704) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=167 (was 142) - SystemLoadAverage LEAK? -, ProcessCount=171 (was 171), AvailableMemoryMB=5383 (was 6309) 2023-11-12T09:28:47,250 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=607 is superior to 500 2023-11-12T09:28:47,265 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleCommand Thread=607, OpenFileDescriptor=709, MaxFileDescriptor=60000, SystemLoadAverage=167, ProcessCount=171, AvailableMemoryMB=5382 2023-11-12T09:28:47,265 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=607 is superior to 500 2023-11-12T09:28:47,265 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(144): create full backup image on multiple tables: command-line 2023-11-12T09:28:47,271 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@235cd975, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:47,274 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:47,276 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:55746, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:47,279 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:28:47,280 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:60426, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:28:47,292 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:28:47,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:28:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781327299 (current time:1699781327299). 2023-11-12T09:28:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:28:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:28:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205cd2ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:47,308 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:55758, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:28:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=133, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:28:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 133 2023-11-12T09:28:47,312 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:28:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=133 2023-11-12T09:28:47,313 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:28:47,315 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:28:47,318 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:28:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=133 2023-11-12T09:28:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=133 2023-11-12T09:28:47,723 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:28:47,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0}] 2023-11-12T09:28:47,725 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:28:47,876 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:47,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=134 2023-11-12T09:28:47,876 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:28:47,877 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-11-12T09:28:47,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/2fb3277523ad45e0a39359b7d56e92cc is 171, key is trslm:hdfs://localhost:34955/backupUT\x00ns2:test-16997811824171/meta:log-roll-map/1699781318778/Put/seqid=0 2023-11-12T09:28:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=133 2023-11-12T09:28:48,284 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/2fb3277523ad45e0a39359b7d56e92cc 2023-11-12T09:28:48,292 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/3ab1f4f4f59e4ddfbd921088b0ee4cb2 is 524, key is session:backup_1699781309433/session:context/1699781319613/Put/seqid=0 2023-11-12T09:28:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=133 2023-11-12T09:28:48,696 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/3ab1f4f4f59e4ddfbd921088b0ee4cb2 2023-11-12T09:28:48,701 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/2fb3277523ad45e0a39359b7d56e92cc as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/2fb3277523ad45e0a39359b7d56e92cc 2023-11-12T09:28:48,706 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/2fb3277523ad45e0a39359b7d56e92cc, entries=6, sequenceid=105, filesize=5.8 K 2023-11-12T09:28:48,707 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/3ab1f4f4f59e4ddfbd921088b0ee4cb2 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/3ab1f4f4f59e4ddfbd921088b0ee4cb2 2023-11-12T09:28:48,712 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/3ab1f4f4f59e4ddfbd921088b0ee4cb2, entries=2, sequenceid=105, filesize=5.5 K 2023-11-12T09:28:48,712 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 835ms, sequenceid=105, compaction requested=true 2023-11-12T09:28:48,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(2520): Flush status journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:28:48,712 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. for snapshot_backup_system completed. 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0703c67cb50e4c9eb5a47d31720493c9, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/2fb3277523ad45e0a39359b7d56e92cc] hfiles 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (1/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (2/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81 for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (3/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (4/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (5/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00 for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (6/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (7/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87 for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (8/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0703c67cb50e4c9eb5a47d31720493c9 for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (9/9): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/2fb3277523ad45e0a39359b7d56e92cc for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/11d8abd27f484bf1b60368d701936880, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/3ab1f4f4f59e4ddfbd921088b0ee4cb2] hfiles 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (1/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (2/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (3/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (4/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (5/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5 for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (6/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8 for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (7/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3 for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (8/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244 for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (9/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/11d8abd27f484bf1b60368d701936880 for snapshot=snapshot_backup_system 2023-11-12T09:28:48,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (10/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/3ab1f4f4f59e4ddfbd921088b0ee4cb2 for snapshot=snapshot_backup_system 2023-11-12T09:28:49,121 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:28:49,121 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=134 2023-11-12T09:28:49,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=134 2023-11-12T09:28:49,122 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 606338a328e82acae51355d485ed81f0 2023-11-12T09:28:49,123 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:28:49,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=134, resume processing ppid=133 2023-11-12T09:28:49,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=134, ppid=133, state=SUCCESS; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 in 1.4010 sec 2023-11-12T09:28:49,125 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:28:49,125 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:28:49,126 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:28:49,126 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:28:49,126 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:28:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=133 2023-11-12T09:28:49,538 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:28:49,551 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:28:49,552 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:28:49,553 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:28:49,553 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 133 2023-11-12T09:28:49,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=133, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2440 sec 2023-11-12T09:28:50,480 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:28:50,518 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997811824171 2023-11-12T09:28:50,518 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997811824171 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:28:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=133 2023-11-12T09:28:51,418 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(115): Backup backup_1699781327271 started at 1699781331418. 2023-11-12T09:28:51,419 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(119): Backup session backup_1699781327271 has been started. 2023-11-12T09:28:51,424 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:28:51,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:28:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:28:51,425 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:28:51,425 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:28:51,426 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:28:51,426 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:28:51,426 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:28:51,428 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:51,428 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:28:51,428 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:28:51,428 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:28:51,428 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:28:51,429 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:28:51,429 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:28:51,429 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:28:51,429 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:28:51,429 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:28:51,429 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:28:51,429 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:28:51,433 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:28:51,433 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:28:51,433 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:28:51,437 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:28:51,437 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:51,437 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:28:51,437 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:28:51,438 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:28:51,438 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:28:51,438 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:28:51,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:51,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:51,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:28:51,440 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:28:51,440 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:28:51,440 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@50596246[Count = 0] remaining members to acquire global barrier 2023-11-12T09:28:51,440 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:28:51,441 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:28:51,441 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:28:51,441 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:28:51,441 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:28:51,442 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:51,442 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:28:51,442 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781311602 highest: 1699781311602 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:51,442 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781311602) roll requested 2023-11-12T09:28:51,454 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781331442, exclude list is [], retry=0 2023-11-12T09:28:51,458 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:28:51,460 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781311602 with entries=17, filesize=5.81 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781331442 2023-11-12T09:28:51,460 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:28:51,460 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781311602 is not closed yet, will try archiving it next time 2023-11-12T09:28:51,460 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781311620) roll requested 2023-11-12T09:28:51,460 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781291112 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781291112 2023-11-12T09:28:51,463 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781311602 not finished, retry = 0 2023-11-12T09:28:51,473 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781331460.meta, exclude list is [], retry=0 2023-11-12T09:28:51,476 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:28:51,477 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781311620.meta with entries=24, filesize=6.28 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781331460.meta 2023-11-12T09:28:51,477 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:28:51,478 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781311620.meta is not closed yet, will try archiving it next time 2023-11-12T09:28:51,482 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:28:51,482 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781331442 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:51,490 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:28:51,490 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36287,1699781172353' in zk 2023-11-12T09:28:51,493 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:28:51,493 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:28:51,493 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:28:51,493 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:51,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:28:51,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:28:51,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:28:51,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:28:51,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:51,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:51,496 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:28:51,496 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:51,496 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:51,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36287,1699781172353': 2023-11-12T09:28:51,497 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:28:51,497 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:28:51,497 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:28:51,497 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:28:51,497 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:28:51,497 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:28:51,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:28:51,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:28:51,499 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:28:51,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:28:51,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:28:51,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:28:51,500 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:28:51,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:28:51,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:28:51,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:51,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:28:51,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:28:51,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:51,503 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:28:51,506 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:28:51,506 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:28:51,506 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:28:51,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:28:51,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:28:51,507 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:28:51,507 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:28:51,554 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16997811824172' 2023-11-12T09:28:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:28:51,610 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } 2023-11-12T09:28:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781331610 (current time:1699781331610). 2023-11-12T09:28:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:28:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781331610_ns2_test-16997811824171 VERSION not specified, setting to 2 2023-11-12T09:28:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9bbda5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:51,619 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:55766, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:28:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=135, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } 2023-11-12T09:28:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 }, snapshot procedure id = 135 2023-11-12T09:28:51,623 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:28:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=135 2023-11-12T09:28:51,624 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:28:51,626 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781331610_ns2_test-16997811824171 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:28:51,628 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:28:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=135 2023-11-12T09:28:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=135 2023-11-12T09:28:52,034 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:28:52,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure d347587386821efa96e22f590d908cde}] 2023-11-12T09:28:52,036 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure d347587386821efa96e22f590d908cde 2023-11-12T09:28:52,187 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:52,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=136 2023-11-12T09:28:52,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:28:52,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.HRegion(2520): Flush status journal for d347587386821efa96e22f590d908cde: 2023-11-12T09:28:52,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. for snapshot_1699781331610_ns2_test-16997811824171 completed. 2023-11-12T09:28:52,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde.' region-info for snapshot=snapshot_1699781331610_ns2_test-16997811824171 2023-11-12T09:28:52,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:28:52,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_] hfiles 2023-11-12T09:28:52,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_ for snapshot=snapshot_1699781331610_ns2_test-16997811824171 2023-11-12T09:28:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=135 2023-11-12T09:28:52,593 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:28:52,593 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=136 2023-11-12T09:28:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=136 2023-11-12T09:28:52,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781331610_ns2_test-16997811824171 on region d347587386821efa96e22f590d908cde 2023-11-12T09:28:52,595 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure d347587386821efa96e22f590d908cde 2023-11-12T09:28:52,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=136, resume processing ppid=135 2023-11-12T09:28:52,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=136, ppid=135, state=SUCCESS; SnapshotRegionProcedure d347587386821efa96e22f590d908cde in 562 msec 2023-11-12T09:28:52,597 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:28:52,598 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:28:52,598 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:28:52,598 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781331610_ns2_test-16997811824171 2023-11-12T09:28:52,599 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781331610_ns2_test-16997811824171 2023-11-12T09:28:52,607 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:28:52,611 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:28:52,612 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781331610_ns2_test-16997811824171 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781331610_ns2_test-16997811824171 2023-11-12T09:28:52,613 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:28:52,613 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 }, snapshot procedure id = 135 2023-11-12T09:28:52,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=135, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1699781331610_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } in 993 msec 2023-11-12T09:28:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=135 2023-11-12T09:28:52,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } 2023-11-12T09:28:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781332729 (current time:1699781332729). 2023-11-12T09:28:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:28:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781332728_ns3_test-16997811824172 VERSION not specified, setting to 2 2023-11-12T09:28:52,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec0fe36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:28:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:28:52,737 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:55776, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:28:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:28:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:28:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=137, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } 2023-11-12T09:28:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 }, snapshot procedure id = 137 2023-11-12T09:28:52,741 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:28:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=137 2023-11-12T09:28:52,742 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:28:52,744 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781332728_ns3_test-16997811824172 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:28:52,747 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:28:52,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=137 2023-11-12T09:28:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=137 2023-11-12T09:28:53,153 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:28:53,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure f50c319a7db3e25c53784414ac224095}] 2023-11-12T09:28:53,155 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure f50c319a7db3e25c53784414ac224095 2023-11-12T09:28:53,306 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:28:53,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=138 2023-11-12T09:28:53,306 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:28:53,308 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.HRegion(2520): Flush status journal for f50c319a7db3e25c53784414ac224095: 2023-11-12T09:28:53,308 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. for snapshot_1699781332728_ns3_test-16997811824172 completed. 2023-11-12T09:28:53,308 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095.' region-info for snapshot=snapshot_1699781332728_ns3_test-16997811824172 2023-11-12T09:28:53,308 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:28:53,308 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-12T09:28:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=137 2023-11-12T09:28:53,714 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:28:53,714 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=138 2023-11-12T09:28:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=138 2023-11-12T09:28:53,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781332728_ns3_test-16997811824172 on region f50c319a7db3e25c53784414ac224095 2023-11-12T09:28:53,715 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure f50c319a7db3e25c53784414ac224095 2023-11-12T09:28:53,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=138, resume processing ppid=137 2023-11-12T09:28:53,717 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:28:53,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=138, ppid=137, state=SUCCESS; SnapshotRegionProcedure f50c319a7db3e25c53784414ac224095 in 563 msec 2023-11-12T09:28:53,718 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:28:53,718 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:28:53,718 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781332728_ns3_test-16997811824172 2023-11-12T09:28:53,719 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781332728_ns3_test-16997811824172 2023-11-12T09:28:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=137 2023-11-12T09:28:54,127 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:28:54,132 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:28:54,132 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781332728_ns3_test-16997811824172 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781332728_ns3_test-16997811824172 2023-11-12T09:28:54,133 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:28:54,133 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 }, snapshot procedure id = 137 2023-11-12T09:28:54,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=137, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1699781332728_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } in 1.3940 sec 2023-11-12T09:28:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=137 2023-11-12T09:28:54,847 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781327271 2023-11-12T09:28:54,847 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:28:54,847 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-12T09:28:54,848 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781327271_ns3:test-16997811824172 2023-11-12T09:28:54,848 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781332728_ns3_test-16997811824172 to hdfs://localhost:34955/backupUT/backup_1699781327271/ns3/test-16997811824172/ 2023-11-12T09:28:54,848 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:28:54,871 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:28:54,871 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_-859651877_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781327271/ns3/test-16997811824172, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781327271/ns3/test-16997811824172/.hbase-snapshot/.tmp/snapshot_1699781332728_ns3_test-16997811824172 2023-11-12T09:28:54,871 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781332728_ns3_test-16997811824172. 2023-11-12T09:28:54,876 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781332728_ns3_test-16997811824172 to hdfs://localhost:34955/backupUT/backup_1699781327271/ns3/test-16997811824172/.hbase-snapshot/.tmp/snapshot_1699781332728_ns3_test-16997811824172 2023-11-12T09:28:55,295 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:55,296 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:55,296 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:55,296 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:56,283 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-3295256728190342407.jar 2023-11-12T09:28:56,283 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:56,283 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:56,284 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:56,284 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:56,284 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:56,285 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:56,285 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:56,285 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:56,286 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:56,286 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:28:56,287 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:28:56,287 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:28:56,288 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:28:56,288 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:28:56,289 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:28:56,289 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:28:56,289 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:28:56,290 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:28:56,290 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:28:56,290 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:28:56,291 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:28:56,291 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:56,292 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:56,292 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:56,292 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:56,293 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:56,293 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:56,294 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:56,338 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781332728_ns3_test-16997811824172' hfile list 2023-11-12T09:28:56,417 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781336381/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:28:56,417 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781336381/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:28:57,457 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:28:57,458 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:28:57,462 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781332728_ns3_test-16997811824172 2023-11-12T09:28:57,462 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781332728_ns3_test-16997811824172 finished. 2023-11-12T09:28:57,462 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781327271_ns2:test-16997811824171 2023-11-12T09:28:57,462 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781331610_ns2_test-16997811824171 to hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171/ 2023-11-12T09:28:57,462 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:28:57,485 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:28:57,485 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_-1619886382_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171/.hbase-snapshot/.tmp/snapshot_1699781331610_ns2_test-16997811824171 2023-11-12T09:28:57,485 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781331610_ns2_test-16997811824171. 2023-11-12T09:28:57,495 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781331610_ns2_test-16997811824171 to hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171/.hbase-snapshot/.tmp/snapshot_1699781331610_ns2_test-16997811824171 2023-11-12T09:28:57,909 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:57,909 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:57,910 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:57,910 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:58,875 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-297027972266411785.jar 2023-11-12T09:28:58,876 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:58,876 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:58,876 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:58,877 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:58,877 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:58,877 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:58,878 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:58,878 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:58,878 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:28:58,879 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:28:58,880 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:28:58,880 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:28:58,880 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:28:58,881 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:28:58,881 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:28:58,881 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:28:58,882 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:28:58,882 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:28:58,882 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:28:58,883 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:28:58,883 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:28:58,884 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:58,884 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:58,885 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:58,885 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:58,885 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:28:58,886 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:58,886 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:28:58,930 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781331610_ns2_test-16997811824171' hfile list 2023-11-12T09:28:58,932 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-12T09:28:58,999 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781338967/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:28:59,000 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781338967/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:28:59,050 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:28:59,104 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_ output=hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_ 2023-11-12T09:28:59,104 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-11-12T09:29:00,030 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:29:00,031 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:29:00,036 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781331610_ns2_test-16997811824171 2023-11-12T09:29:00,036 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781331610_ns2_test-16997811824171 finished. 2023-11-12T09:29:00,036 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): ns3:test-16997811824172 2023-11-12T09:29:00,037 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): ns2:test-16997811824171 2023-11-12T09:29:00,043 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781327271 2023-11-12T09:29:00,043 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:29:00,448 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781327271/.backup.manifest 2023-11-12T09:29:00,448 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781327271 2023-11-12T09:29:00,448 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:29:00,853 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781327271/.backup.manifest 2023-11-12T09:29:00,853 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(358): Backup backup_1699781327271 finished: type=FULL,tablelist=ns3:test-16997811824172;ns2:test-16997811824171,targetRootDir=hdfs://localhost:34955/backupUT,startts=1699781331418,completets=1699781340043,bytescopied=0 2023-11-12T09:29:00,853 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:29:00,853 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781332728_ns3_test-16997811824172 2023-11-12T09:29:00,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781332728_ns3_test-16997811824172" 2023-11-12T09:29:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781332728_ns3_test-16997811824172 2023-11-12T09:29:00,857 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781332728_ns3_test-16997811824172 for backup backup_1699781327271 succeeded. 2023-11-12T09:29:00,857 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781331610_ns2_test-16997811824171 2023-11-12T09:29:00,858 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781331610_ns2_test-16997811824171" 2023-11-12T09:29:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781331610_ns2_test-16997811824171 2023-11-12T09:29:00,860 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781331610_ns2_test-16997811824171 for backup backup_1699781327271 succeeded. 2023-11-12T09:29:00,861 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:29:00,865 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:29:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:29:00,867 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:29:00,868 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:29:00,873 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(376): Backup backup_1699781327271 completed. 2023-11-12T09:29:00,874 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:00,891 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775a7e88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:00,892 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:00,894 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:48746, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:00,894 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-11-12T09:29:00,895 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-11-12T09:29:00,895 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781327271/.backup.manifest 2023-11-12T09:29:00,897 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781327271/.backup.manifest 2023-11-12T09:29:00,898 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781327271/.backup.manifest 2023-11-12T09:29:00,899 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781327271/.backup.manifest 2023-11-12T09:29:00,900 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781327271/.backup.manifest 2023-11-12T09:29:00,901 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781327271/.backup.manifest 2023-11-12T09:29:00,901 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16997811824171' to 'ns2:table2' from full backup image hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171 2023-11-12T09:29:00,907 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(487): Creating target table 'ns2:table2' 2023-11-12T09:29:00,907 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde 2023-11-12T09:29:00,908 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f in region [hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde] 2023-11-12T09:29:00,910 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_ first=row0 last=row98 2023-11-12T09:29:00,911 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:00,913 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:44568, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:00,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:29:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:table2 2023-11-12T09:29:00,916 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:29:00,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 139 2023-11-12T09:29:00,917 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:29:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=139 2023-11-12T09:29:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=139 2023-11-12T09:29:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=139 2023-11-12T09:29:01,323 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 537cee01b77e55e3fd9803383ce6b521, NAME => 'ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:29:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=139 2023-11-12T09:29:01,729 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:29:01,729 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1668): Closing 537cee01b77e55e3fd9803383ce6b521, disabling compactions & flushes 2023-11-12T09:29:01,729 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521. 2023-11-12T09:29:01,729 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521. 2023-11-12T09:29:01,729 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521. after waiting 0 ms 2023-11-12T09:29:01,729 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521. 2023-11-12T09:29:01,729 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1909): Closed ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521. 2023-11-12T09:29:01,729 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1622): Region close journal for 537cee01b77e55e3fd9803383ce6b521: 2023-11-12T09:29:01,730 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:29:01,731 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1699781341730"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781341730"}]},"ts":"1699781341730"} 2023-11-12T09:29:01,732 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:29:01,733 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:29:01,733 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781341733"}]},"ts":"1699781341733"} 2023-11-12T09:29:01,734 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2023-11-12T09:29:01,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=537cee01b77e55e3fd9803383ce6b521, ASSIGN}] 2023-11-12T09:29:01,746 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=537cee01b77e55e3fd9803383ce6b521, ASSIGN 2023-11-12T09:29:01,747 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:table2, region=537cee01b77e55e3fd9803383ce6b521, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:29:01,898 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=537cee01b77e55e3fd9803383ce6b521, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:01,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE; OpenRegionProcedure 537cee01b77e55e3fd9803383ce6b521, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:29:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=139 2023-11-12T09:29:02,051 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:02,054 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(130): Open ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521. 2023-11-12T09:29:02,054 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7254): Opening region: {ENCODED => 537cee01b77e55e3fd9803383ce6b521, NAME => 'ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:29:02,054 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:29:02,055 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 537cee01b77e55e3fd9803383ce6b521 2023-11-12T09:29:02,055 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(888): Instantiated ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:29:02,055 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7296): checking encryption for 537cee01b77e55e3fd9803383ce6b521 2023-11-12T09:29:02,055 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7299): checking classloading for 537cee01b77e55e3fd9803383ce6b521 2023-11-12T09:29:02,056 INFO [StoreOpener-537cee01b77e55e3fd9803383ce6b521-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 537cee01b77e55e3fd9803383ce6b521 2023-11-12T09:29:02,057 INFO [StoreOpener-537cee01b77e55e3fd9803383ce6b521-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 537cee01b77e55e3fd9803383ce6b521 columnFamilyName f 2023-11-12T09:29:02,057 DEBUG [StoreOpener-537cee01b77e55e3fd9803383ce6b521-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:02,058 INFO [StoreOpener-537cee01b77e55e3fd9803383ce6b521-1 {}] regionserver.HStore(324): Store=537cee01b77e55e3fd9803383ce6b521/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:29:02,058 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521 2023-11-12T09:29:02,059 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521 2023-11-12T09:29:02,060 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1076): writing seq id for 537cee01b77e55e3fd9803383ce6b521 2023-11-12T09:29:02,062 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:29:02,062 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): Opened 537cee01b77e55e3fd9803383ce6b521; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10841983200, jitterRate=0.009738370776176453}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:29:02,063 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(992): Region open journal for 537cee01b77e55e3fd9803383ce6b521: 2023-11-12T09:29:02,063 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521., pid=141, masterSystemTime=1699781342051 2023-11-12T09:29:02,065 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521. 2023-11-12T09:29:02,065 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(158): Opened ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521. 2023-11-12T09:29:02,065 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=537cee01b77e55e3fd9803383ce6b521, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:02,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=141, resume processing ppid=140 2023-11-12T09:29:02,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=141, ppid=140, state=SUCCESS; OpenRegionProcedure 537cee01b77e55e3fd9803383ce6b521, server=jenkins-hbase3.apache.org,36287,1699781172353 in 166 msec 2023-11-12T09:29:02,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=140, resume processing ppid=139 2023-11-12T09:29:02,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=140, ppid=139, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=537cee01b77e55e3fd9803383ce6b521, ASSIGN in 323 msec 2023-11-12T09:29:02,069 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:29:02,069 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781342069"}]},"ts":"1699781342069"} 2023-11-12T09:29:02,070 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2023-11-12T09:29:02,074 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:29:02,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=139, state=SUCCESS; CreateTableProcedure table=ns2:table2 in 1.1600 sec 2023-11-12T09:29:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=139 2023-11-12T09:29:03,027 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde from hbase tables ns2:test-16997811824171 to tables ns2:table2 2023-11-12T09:29:03,027 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16997811824171 into ns2:table2 2023-11-12T09:29:03,029 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781343027 from hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde 2023-11-12T09:29:03,034 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8d5fd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:03,035 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:03,036 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:44574, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:03,036 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:29:03,036 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:table2 2023-11-12T09:29:03,039 DEBUG [hconnection-0x7b8d7cdf-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:03,040 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:48758, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:03,042 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:table2: 2023-11-12T09:29:03,042 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-12T09:29:03,042 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/partitions_1136ec98-e128-40f5-873b-62aa4c2e4b4b 2023-11-12T09:29:03,451 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:03,451 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:03,452 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:03,452 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:03,555 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2023-11-12T09:29:04,411 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-4284003006611489474.jar 2023-11-12T09:29:04,412 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,412 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,413 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,413 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,413 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,414 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,414 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,414 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,415 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,415 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:04,416 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:29:04,416 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:29:04,417 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:29:04,417 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:29:04,418 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:29:04,418 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:29:04,418 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:29:04,419 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:29:04,419 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:29:04,420 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:29:04,420 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:29:04,420 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,421 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,421 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,421 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,421 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,422 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:04,422 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:04,423 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:table2 2023-11-12T09:29:04,423 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:04,423 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-12T09:29:04,423 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-12T09:29:04,424 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:04,435 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-12T09:29:04,515 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781344480/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:29:04,515 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781344480/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:29:04,565 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_ 2023-11-12T09:29:04,569 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-12T09:29:04,588 DEBUG [pool-485-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d294a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:04,589 DEBUG [hconnection-0x3a164d6c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:04,591 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:48768, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:04,594 DEBUG [pool-485-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:04,594 DEBUG [pool-485-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-12T09:29:04,594 DEBUG [pool-485-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:29:04,599 DEBUG [pool-485-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781343027/_temporary/0/_temporary/attempt_local1578917057_0022_r_000000_0/f/86f8a2fe05664458abbaf7626a8d6b94 is 37, key is row10/f:q1/1699781187295/Put/seqid=0 2023-11-12T09:29:04,600 WARN [Thread-2116 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36287] Specified favored nodes: [jenkins-hbase3.apache.org:36287] 2023-11-12T09:29:05,562 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781343027 2023-11-12T09:29:05,567 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13421f68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:05,569 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:05,571 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46972, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:05,576 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781343027/_SUCCESS 2023-11-12T09:29:05,579 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:05,581 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45228, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:05,589 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781343027/f/86f8a2fe05664458abbaf7626a8d6b94 first=Optional[row0] last=Optional[row98] 2023-11-12T09:29:05,590 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521., hostname=jenkins-hbase3.apache.org,36287,1699781172353, seqNum=2 for row with hfile group [{f,hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781343027/f/86f8a2fe05664458abbaf7626a8d6b94}] 2023-11-12T09:29:05,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781343027/f/86f8a2fe05664458abbaf7626a8d6b94 for inclusion in 537cee01b77e55e3fd9803383ce6b521/f 2023-11-12T09:29:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-12T09:29:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-12T09:29:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegion(2520): Flush status journal for 537cee01b77e55e3fd9803383ce6b521: 2023-11-12T09:29:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-ns2-table2-1699781343027/f/86f8a2fe05664458abbaf7626a8d6b94 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_table2__sbp1q65nj714a578j3ujvv0rshtll6a95cm93koqimor4n6v5s4kj3kjkkie7ts/f/86f8a2fe05664458abbaf7626a8d6b94 2023-11-12T09:29:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_table2__sbp1q65nj714a578j3ujvv0rshtll6a95cm93koqimor4n6v5s4kj3kjkkie7ts/f/86f8a2fe05664458abbaf7626a8d6b94 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521/f/4a45e4f311f74a2f9731e1b48a362d59_SeqId_4_ 2023-11-12T09:29:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749a4e13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:05,607 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45240, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:05,612 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:46988, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:05,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_table2__sbp1q65nj714a578j3ujvv0rshtll6a95cm93koqimor4n6v5s4kj3kjkkie7ts/f/86f8a2fe05664458abbaf7626a8d6b94 into 537cee01b77e55e3fd9803383ce6b521/f as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521/f/4a45e4f311f74a2f9731e1b48a362d59_SeqId_4_ - updating store file list. 2023-11-12T09:29:05,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521/f/4a45e4f311f74a2f9731e1b48a362d59_SeqId_4_ into 537cee01b77e55e3fd9803383ce6b521/f 2023-11-12T09:29:05,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_table2__sbp1q65nj714a578j3ujvv0rshtll6a95cm93koqimor4n6v5s4kj3kjkkie7ts/f/86f8a2fe05664458abbaf7626a8d6b94 into 537cee01b77e55e3fd9803383ce6b521/f (new location: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521/f/4a45e4f311f74a2f9731e1b48a362d59_SeqId_4_) 2023-11-12T09:29:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__ns2_table2__sbp1q65nj714a578j3ujvv0rshtll6a95cm93koqimor4n6v5s4kj3kjkkie7ts/f/86f8a2fe05664458abbaf7626a8d6b94 2023-11-12T09:29:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd469c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:05,637 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45250, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:29:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:05,641 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47004, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-12T09:29:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:05,654 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:05,654 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-12T09:29:05,654 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:29:05,654 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(233): Backup: backup_1699781327271 hdfs://localhost:34955/backupUT/backup_1699781327271/ns2/test-16997811824171/ 2023-11-12T09:29:05,654 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781327271/.backup.manifest 2023-11-12T09:29:05,658 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781327271/.backup.manifest 2023-11-12T09:29:05,658 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16997811824172' to 'ns3:table3_restore' from full backup image hdfs://localhost:34955/backupUT/backup_1699781327271/ns3/test-16997811824172 2023-11-12T09:29:05,663 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:34955/backupUT/backup_1699781327271/ns3/test-16997811824172/archive/data/ns3/test-16997811824172 does not exists 2023-11-12T09:29:05,663 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16997811824172, will only create table 2023-11-12T09:29:05,663 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(487): Creating target table 'ns3:table3_restore' 2023-11-12T09:29:05,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:29:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:table3_restore 2023-11-12T09:29:05,667 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:29:05,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "table3_restore" procId is: 142 2023-11-12T09:29:05,667 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:29:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2023-11-12T09:29:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2023-11-12T09:29:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2023-11-12T09:29:06,074 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 865c437b158291a1ca93bfed5347da85, NAME => 'ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:29:06,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2023-11-12T09:29:06,480 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:29:06,480 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1668): Closing 865c437b158291a1ca93bfed5347da85, disabling compactions & flushes 2023-11-12T09:29:06,480 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85. 2023-11-12T09:29:06,480 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85. 2023-11-12T09:29:06,480 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85. after waiting 0 ms 2023-11-12T09:29:06,480 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85. 2023-11-12T09:29:06,480 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85. 2023-11-12T09:29:06,480 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1622): Region close journal for 865c437b158291a1ca93bfed5347da85: 2023-11-12T09:29:06,481 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:29:06,481 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1699781346481"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781346481"}]},"ts":"1699781346481"} 2023-11-12T09:29:06,482 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:29:06,483 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:29:06,483 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781346483"}]},"ts":"1699781346483"} 2023-11-12T09:29:06,484 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLING in hbase:meta 2023-11-12T09:29:06,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=865c437b158291a1ca93bfed5347da85, ASSIGN}] 2023-11-12T09:29:06,490 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=865c437b158291a1ca93bfed5347da85, ASSIGN 2023-11-12T09:29:06,490 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:table3_restore, region=865c437b158291a1ca93bfed5347da85, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:29:06,642 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=143 updating hbase:meta row=865c437b158291a1ca93bfed5347da85, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:06,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE; OpenRegionProcedure 865c437b158291a1ca93bfed5347da85, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:29:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2023-11-12T09:29:06,795 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:06,798 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(130): Open ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85. 2023-11-12T09:29:06,798 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7254): Opening region: {ENCODED => 865c437b158291a1ca93bfed5347da85, NAME => 'ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:29:06,799 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:29:06,799 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table3_restore 865c437b158291a1ca93bfed5347da85 2023-11-12T09:29:06,799 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:29:06,799 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7296): checking encryption for 865c437b158291a1ca93bfed5347da85 2023-11-12T09:29:06,799 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7299): checking classloading for 865c437b158291a1ca93bfed5347da85 2023-11-12T09:29:06,800 INFO [StoreOpener-865c437b158291a1ca93bfed5347da85-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 865c437b158291a1ca93bfed5347da85 2023-11-12T09:29:06,801 INFO [StoreOpener-865c437b158291a1ca93bfed5347da85-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 865c437b158291a1ca93bfed5347da85 columnFamilyName f 2023-11-12T09:29:06,801 DEBUG [StoreOpener-865c437b158291a1ca93bfed5347da85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:06,802 INFO [StoreOpener-865c437b158291a1ca93bfed5347da85-1 {}] regionserver.HStore(324): Store=865c437b158291a1ca93bfed5347da85/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:29:06,802 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/865c437b158291a1ca93bfed5347da85 2023-11-12T09:29:06,803 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/865c437b158291a1ca93bfed5347da85 2023-11-12T09:29:06,804 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1076): writing seq id for 865c437b158291a1ca93bfed5347da85 2023-11-12T09:29:06,806 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/865c437b158291a1ca93bfed5347da85/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:29:06,806 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): Opened 865c437b158291a1ca93bfed5347da85; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10539670880, jitterRate=-0.01841665804386139}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:29:06,807 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(992): Region open journal for 865c437b158291a1ca93bfed5347da85: 2023-11-12T09:29:06,807 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85., pid=144, masterSystemTime=1699781346795 2023-11-12T09:29:06,809 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85. 2023-11-12T09:29:06,809 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(158): Opened ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85. 2023-11-12T09:29:06,809 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=143 updating hbase:meta row=865c437b158291a1ca93bfed5347da85, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:06,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=144, resume processing ppid=143 2023-11-12T09:29:06,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=144, ppid=143, state=SUCCESS; OpenRegionProcedure 865c437b158291a1ca93bfed5347da85, server=jenkins-hbase3.apache.org,36287,1699781172353 in 166 msec 2023-11-12T09:29:06,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=143, resume processing ppid=142 2023-11-12T09:29:06,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=143, ppid=142, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=865c437b158291a1ca93bfed5347da85, ASSIGN in 323 msec 2023-11-12T09:29:06,812 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:29:06,813 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781346812"}]},"ts":"1699781346812"} 2023-11-12T09:29:06,813 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLED in hbase:meta 2023-11-12T09:29:06,815 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:29:06,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=142, state=SUCCESS; CreateTableProcedure table=ns3:table3_restore in 1.1510 sec 2023-11-12T09:29:07,594 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:29:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2023-11-12T09:29:07,775 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:29:07,775 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(233): Backup: backup_1699781327271 hdfs://localhost:34955/backupUT/backup_1699781327271/ns3/test-16997811824172/ 2023-11-12T09:29:07,775 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-12T09:29:07,776 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:07,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable ns2:table2 2023-11-12T09:29:07,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:table2 2023-11-12T09:29:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=145 2023-11-12T09:29:07,786 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781347786"}]},"ts":"1699781347786"} 2023-11-12T09:29:07,787 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2023-11-12T09:29:07,789 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set ns2:table2 to state=DISABLING 2023-11-12T09:29:07,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=537cee01b77e55e3fd9803383ce6b521, UNASSIGN}] 2023-11-12T09:29:07,791 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=537cee01b77e55e3fd9803383ce6b521, UNASSIGN 2023-11-12T09:29:07,791 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=146 updating hbase:meta row=537cee01b77e55e3fd9803383ce6b521, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:07,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE; CloseRegionProcedure 537cee01b77e55e3fd9803383ce6b521, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:29:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=145 2023-11-12T09:29:07,943 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:07,944 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(119): Close 537cee01b77e55e3fd9803383ce6b521 2023-11-12T09:29:07,944 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1668): Closing 537cee01b77e55e3fd9803383ce6b521, disabling compactions & flushes 2023-11-12T09:29:07,944 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1690): Closing region ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521. 2023-11-12T09:29:07,944 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521. 2023-11-12T09:29:07,944 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521. after waiting 0 ms 2023-11-12T09:29:07,944 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521. 2023-11-12T09:29:07,950 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-12T09:29:07,951 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:07,952 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1909): Closed ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521. 2023-11-12T09:29:07,952 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1622): Region close journal for 537cee01b77e55e3fd9803383ce6b521: 2023-11-12T09:29:07,953 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(163): Closed 537cee01b77e55e3fd9803383ce6b521 2023-11-12T09:29:07,953 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=146 updating hbase:meta row=537cee01b77e55e3fd9803383ce6b521, regionState=CLOSED 2023-11-12T09:29:07,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=147, resume processing ppid=146 2023-11-12T09:29:07,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=147, ppid=146, state=SUCCESS; CloseRegionProcedure 537cee01b77e55e3fd9803383ce6b521, server=jenkins-hbase3.apache.org,36287,1699781172353 in 162 msec 2023-11-12T09:29:07,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=146, resume processing ppid=145 2023-11-12T09:29:07,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=146, ppid=145, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=537cee01b77e55e3fd9803383ce6b521, UNASSIGN in 166 msec 2023-11-12T09:29:07,957 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781347957"}]},"ts":"1699781347957"} 2023-11-12T09:29:07,958 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2023-11-12T09:29:07,960 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set ns2:table2 to state=DISABLED 2023-11-12T09:29:07,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=145, state=SUCCESS; DisableTableProcedure table=ns2:table2 in 182 msec 2023-11-12T09:29:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=145 2023-11-12T09:29:08,086 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$5(2449): Client=jenkins//172.31.12.81 delete ns2:table2 2023-11-12T09:29:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns2:table2 2023-11-12T09:29:08,089 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-12T09:29:08,089 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=148, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-12T09:29:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=148 2023-11-12T09:29:08,091 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521 2023-11-12T09:29:08,093 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521/f, FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521/recovered.edits] 2023-11-12T09:29:08,097 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521/f/4a45e4f311f74a2f9731e1b48a362d59_SeqId_4_ to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521/f/4a45e4f311f74a2f9731e1b48a362d59_SeqId_4_ 2023-11-12T09:29:08,099 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521/recovered.edits/6.seqid to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521/recovered.edits/6.seqid 2023-11-12T09:29:08,100 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/table2/537cee01b77e55e3fd9803383ce6b521 2023-11-12T09:29:08,100 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived ns2:table2 regions 2023-11-12T09:29:08,102 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=148, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-12T09:29:08,103 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2023-11-12T09:29:08,105 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:table2' descriptor. 2023-11-12T09:29:08,106 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=148, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-12T09:29:08,106 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:table2' from region states. 2023-11-12T09:29:08,106 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781348106"}]},"ts":"9223372036854775807"} 2023-11-12T09:29:08,107 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:29:08,107 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 537cee01b77e55e3fd9803383ce6b521, NAME => 'ns2:table2,,1699781340913.537cee01b77e55e3fd9803383ce6b521.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:29:08,107 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:table2' as deleted. 2023-11-12T09:29:08,107 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781348107"}]},"ts":"9223372036854775807"} 2023-11-12T09:29:08,109 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:table2 state from META 2023-11-12T09:29:08,111 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=148, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-12T09:29:08,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=148, state=SUCCESS; DeleteTableProcedure table=ns2:table2 in 25 msec 2023-11-12T09:29:08,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=148 2023-11-12T09:29:08,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable ns3:table3_restore 2023-11-12T09:29:08,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=149, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:table3_restore 2023-11-12T09:29:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=149 2023-11-12T09:29:08,196 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781348195"}]},"ts":"1699781348195"} 2023-11-12T09:29:08,196 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLING in hbase:meta 2023-11-12T09:29:08,199 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set ns3:table3_restore to state=DISABLING 2023-11-12T09:29:08,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=865c437b158291a1ca93bfed5347da85, UNASSIGN}] 2023-11-12T09:29:08,201 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=865c437b158291a1ca93bfed5347da85, UNASSIGN 2023-11-12T09:29:08,202 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=150 updating hbase:meta row=865c437b158291a1ca93bfed5347da85, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:08,203 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE; CloseRegionProcedure 865c437b158291a1ca93bfed5347da85, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:29:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=149 2023-11-12T09:29:08,354 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:08,354 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(119): Close 865c437b158291a1ca93bfed5347da85 2023-11-12T09:29:08,356 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1668): Closing 865c437b158291a1ca93bfed5347da85, disabling compactions & flushes 2023-11-12T09:29:08,356 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85. 2023-11-12T09:29:08,356 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85. 2023-11-12T09:29:08,356 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85. after waiting 0 ms 2023-11-12T09:29:08,356 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85. 2023-11-12T09:29:08,359 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/865c437b158291a1ca93bfed5347da85/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:29:08,360 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:08,361 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85. 2023-11-12T09:29:08,361 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1622): Region close journal for 865c437b158291a1ca93bfed5347da85: 2023-11-12T09:29:08,362 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(163): Closed 865c437b158291a1ca93bfed5347da85 2023-11-12T09:29:08,362 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=150 updating hbase:meta row=865c437b158291a1ca93bfed5347da85, regionState=CLOSED 2023-11-12T09:29:08,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=151, resume processing ppid=150 2023-11-12T09:29:08,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=151, ppid=150, state=SUCCESS; CloseRegionProcedure 865c437b158291a1ca93bfed5347da85, server=jenkins-hbase3.apache.org,36287,1699781172353 in 161 msec 2023-11-12T09:29:08,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=150, resume processing ppid=149 2023-11-12T09:29:08,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=150, ppid=149, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=865c437b158291a1ca93bfed5347da85, UNASSIGN in 164 msec 2023-11-12T09:29:08,366 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781348366"}]},"ts":"1699781348366"} 2023-11-12T09:29:08,367 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLED in hbase:meta 2023-11-12T09:29:08,368 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set ns3:table3_restore to state=DISABLED 2023-11-12T09:29:08,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=149, state=SUCCESS; DisableTableProcedure table=ns3:table3_restore in 177 msec 2023-11-12T09:29:08,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=149 2023-11-12T09:29:08,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$5(2449): Client=jenkins//172.31.12.81 delete ns3:table3_restore 2023-11-12T09:29:08,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns3:table3_restore 2023-11-12T09:29:08,500 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-12T09:29:08,501 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=152, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-12T09:29:08,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=152 2023-11-12T09:29:08,503 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/865c437b158291a1ca93bfed5347da85 2023-11-12T09:29:08,504 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/865c437b158291a1ca93bfed5347da85/f, FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/865c437b158291a1ca93bfed5347da85/recovered.edits] 2023-11-12T09:29:08,509 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/865c437b158291a1ca93bfed5347da85/recovered.edits/4.seqid to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/ns3/table3_restore/865c437b158291a1ca93bfed5347da85/recovered.edits/4.seqid 2023-11-12T09:29:08,509 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/table3_restore/865c437b158291a1ca93bfed5347da85 2023-11-12T09:29:08,509 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived ns3:table3_restore regions 2023-11-12T09:29:08,511 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=152, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-12T09:29:08,513 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:table3_restore from hbase:meta 2023-11-12T09:29:08,514 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:table3_restore' descriptor. 2023-11-12T09:29:08,515 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=152, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-12T09:29:08,515 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:table3_restore' from region states. 2023-11-12T09:29:08,515 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781348515"}]},"ts":"9223372036854775807"} 2023-11-12T09:29:08,516 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:29:08,516 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 865c437b158291a1ca93bfed5347da85, NAME => 'ns3:table3_restore,,1699781345664.865c437b158291a1ca93bfed5347da85.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:29:08,517 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:table3_restore' as deleted. 2023-11-12T09:29:08,517 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781348517"}]},"ts":"9223372036854775807"} 2023-11-12T09:29:08,518 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:table3_restore state from META 2023-11-12T09:29:08,520 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=152, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-12T09:29:08,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=152, state=SUCCESS; DeleteTableProcedure table=ns3:table3_restore in 22 msec 2023-11-12T09:29:08,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=152 2023-11-12T09:29:08,623 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleCommand Thread=605 (was 607), OpenFileDescriptor=708 (was 709), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=164 (was 167), ProcessCount=171 (was 171), AvailableMemoryMB=5319 (was 5382) 2023-11-12T09:29:08,623 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-12T09:29:08,638 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleDNE Thread=605, OpenFileDescriptor=708, MaxFileDescriptor=60000, SystemLoadAverage=164, ProcessCount=171, AvailableMemoryMB=5318 2023-11-12T09:29:08,638 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-12T09:29:08,639 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(260): test restore fails on a single table that does not exist 2023-11-12T09:29:08,644 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40bdd835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:08,646 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:08,648 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47010, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:08,650 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:08,651 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45252, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:08,660 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:29:08,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781348665 (current time:1699781348665). 2023-11-12T09:29:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:29:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217ccbb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:08,674 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47012, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:08,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:08,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=153, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:08,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 153 2023-11-12T09:29:08,678 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-12T09:29:08,678 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:08,681 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:08,683 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-12T09:29:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-12T09:29:09,090 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:09,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0}] 2023-11-12T09:29:09,092 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:29:09,243 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:09,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=154 2023-11-12T09:29:09,243 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:29:09,244 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-11-12T09:29:09,248 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/5d7a351cc7434a339a200f712867b7ff is 171, key is trslm:hdfs://localhost:34955/backupUT\x00ns2:test-16997811824171/meta:log-roll-map/1699781340039/Put/seqid=0 2023-11-12T09:29:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-12T09:29:09,652 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/5d7a351cc7434a339a200f712867b7ff 2023-11-12T09:29:09,659 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/1615bcf1a8bf41eab0b3ea9ba058fb43 is 524, key is session:backup_1699781327271/session:context/1699781340868/Put/seqid=0 2023-11-12T09:29:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-12T09:29:10,063 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/1615bcf1a8bf41eab0b3ea9ba058fb43 2023-11-12T09:29:10,069 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/5d7a351cc7434a339a200f712867b7ff as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/5d7a351cc7434a339a200f712867b7ff 2023-11-12T09:29:10,074 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/5d7a351cc7434a339a200f712867b7ff, entries=6, sequenceid=116, filesize=5.8 K 2023-11-12T09:29:10,075 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/1615bcf1a8bf41eab0b3ea9ba058fb43 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/1615bcf1a8bf41eab0b3ea9ba058fb43 2023-11-12T09:29:10,079 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/1615bcf1a8bf41eab0b3ea9ba058fb43, entries=2, sequenceid=116, filesize=5.5 K 2023-11-12T09:29:10,080 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 836ms, sequenceid=116, compaction requested=true 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(2520): Flush status journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. for snapshot_backup_system completed. 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0703c67cb50e4c9eb5a47d31720493c9, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/2fb3277523ad45e0a39359b7d56e92cc, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/5d7a351cc7434a339a200f712867b7ff] hfiles 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (1/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db for snapshot=snapshot_backup_system 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (2/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81 for snapshot=snapshot_backup_system 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (3/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc for snapshot=snapshot_backup_system 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (4/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd for snapshot=snapshot_backup_system 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (5/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00 for snapshot=snapshot_backup_system 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (6/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa for snapshot=snapshot_backup_system 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (7/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87 for snapshot=snapshot_backup_system 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (8/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0703c67cb50e4c9eb5a47d31720493c9 for snapshot=snapshot_backup_system 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (9/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/2fb3277523ad45e0a39359b7d56e92cc for snapshot=snapshot_backup_system 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (10/10): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/5d7a351cc7434a339a200f712867b7ff for snapshot=snapshot_backup_system 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/11d8abd27f484bf1b60368d701936880, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/3ab1f4f4f59e4ddfbd921088b0ee4cb2, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/1615bcf1a8bf41eab0b3ea9ba058fb43] hfiles 2023-11-12T09:29:10,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (1/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 for snapshot=snapshot_backup_system 2023-11-12T09:29:10,081 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (2/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b for snapshot=snapshot_backup_system 2023-11-12T09:29:10,081 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (3/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd for snapshot=snapshot_backup_system 2023-11-12T09:29:10,081 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (4/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c for snapshot=snapshot_backup_system 2023-11-12T09:29:10,081 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (5/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5 for snapshot=snapshot_backup_system 2023-11-12T09:29:10,081 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (6/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8 for snapshot=snapshot_backup_system 2023-11-12T09:29:10,081 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (7/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3 for snapshot=snapshot_backup_system 2023-11-12T09:29:10,081 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (8/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244 for snapshot=snapshot_backup_system 2023-11-12T09:29:10,081 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (9/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/11d8abd27f484bf1b60368d701936880 for snapshot=snapshot_backup_system 2023-11-12T09:29:10,081 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (10/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/3ab1f4f4f59e4ddfbd921088b0ee4cb2 for snapshot=snapshot_backup_system 2023-11-12T09:29:10,081 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (11/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/1615bcf1a8bf41eab0b3ea9ba058fb43 for snapshot=snapshot_backup_system 2023-11-12T09:29:10,486 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:29:10,486 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=154 2023-11-12T09:29:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=154 2023-11-12T09:29:10,486 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 606338a328e82acae51355d485ed81f0 2023-11-12T09:29:10,488 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:29:10,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=154, resume processing ppid=153 2023-11-12T09:29:10,489 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:10,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=154, ppid=153, state=SUCCESS; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 in 1.3980 sec 2023-11-12T09:29:10,490 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:10,490 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:10,490 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:29:10,491 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:29:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-12T09:29:10,904 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:10,916 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:10,917 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:29:10,918 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:10,918 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 153 2023-11-12T09:29:10,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=153, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2430 sec 2023-11-12T09:29:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-12T09:29:12,785 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(115): Backup backup_1699781348644 started at 1699781352785. 2023-11-12T09:29:12,786 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(119): Backup session backup_1699781348644 has been started. 2023-11-12T09:29:12,792 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:29:12,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:29:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:29:12,793 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:29:12,793 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:29:12,794 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:29:12,794 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:29:12,795 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:12,796 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:12,796 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:12,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:12,796 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:29:12,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:12,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:29:12,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:12,797 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:29:12,797 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:29:12,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:29:12,797 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:29:12,797 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:29:12,798 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:29:12,798 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:29:12,798 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:29:12,800 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:12,800 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:12,800 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:12,800 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:12,801 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:29:12,801 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:12,801 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:12,801 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:12,801 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:12,802 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:12,802 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:29:12,802 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:29:12,802 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7ea68115[Count = 0] remaining members to acquire global barrier 2023-11-12T09:29:12,802 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:12,804 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:12,804 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:12,804 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:29:12,804 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:29:12,804 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:29:12,804 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:12,804 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781331442 highest: 1699781331442 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:12,804 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781331442) roll requested 2023-11-12T09:29:12,817 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781352804, exclude list is [], retry=0 2023-11-12T09:29:12,822 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:29:12,823 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781331442 with entries=15, filesize=4.05 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781352804 2023-11-12T09:29:12,824 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:29:12,824 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781331442 is not closed yet, will try archiving it next time 2023-11-12T09:29:12,824 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781331460) roll requested 2023-11-12T09:29:12,842 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781352824.meta, exclude list is [], retry=0 2023-11-12T09:29:12,845 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-11-12T09:29:12,845 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781352804 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:12,846 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:29:12,847 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781331460.meta with entries=24, filesize=5.77 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781352824.meta 2023-11-12T09:29:12,847 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:29:12,847 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781331460.meta is not closed yet, will try archiving it next time 2023-11-12T09:29:12,854 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:29:12,854 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36287,1699781172353' in zk 2023-11-12T09:29:12,858 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:29:12,858 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:12,858 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:29:12,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:12,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:12,859 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:12,860 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:12,860 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:12,860 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:12,860 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:12,861 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:12,861 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:12,861 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:12,862 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36287,1699781172353': 2023-11-12T09:29:12,862 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:29:12,862 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:29:12,862 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:29:12,862 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:29:12,862 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:29:12,862 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:29:12,864 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:29:12,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:12,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:12,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:12,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:12,865 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:12,865 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:12,865 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:12,865 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:12,865 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:12,866 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:12,866 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:12,866 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:12,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:12,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:12,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:12,872 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:12,872 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:12,872 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:12,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:29:12,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:29:12,873 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:12,873 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:29:12,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:29:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781352976 (current time:1699781352976). 2023-11-12T09:29:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781352975_default_test-1699781182417 VERSION not specified, setting to 2 2023-11-12T09:29:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477f9946, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:12,991 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47020, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=155, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:29:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 155 2023-11-12T09:29:12,997 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=155 2023-11-12T09:29:12,998 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:13,000 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781352975_default_test-1699781182417 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:13,003 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=155 2023-11-12T09:29:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=155 2023-11-12T09:29:13,408 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:13,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure 3ca9201e34d84739ee34b92f9445e4a2}] 2023-11-12T09:29:13,410 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:29:13,453 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:29:13,561 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:13,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=156 2023-11-12T09:29:13,562 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:29:13,562 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.HRegion(2520): Flush status journal for 3ca9201e34d84739ee34b92f9445e4a2: 2023-11-12T09:29:13,562 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. for snapshot_1699781352975_default_test-1699781182417 completed. 2023-11-12T09:29:13,562 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(240): Storing 'test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2.' region-info for snapshot=snapshot_1699781352975_default_test-1699781182417 2023-11-12T09:29:13,562 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:13,562 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_] hfiles 2023-11-12T09:29:13,562 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_ for snapshot=snapshot_1699781352975_default_test-1699781182417 2023-11-12T09:29:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=155 2023-11-12T09:29:13,968 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:29:13,968 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=156 2023-11-12T09:29:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=156 2023-11-12T09:29:13,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781352975_default_test-1699781182417 on region 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:29:13,970 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:29:13,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=156, resume processing ppid=155 2023-11-12T09:29:13,973 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=156, ppid=155, state=SUCCESS; SnapshotRegionProcedure 3ca9201e34d84739ee34b92f9445e4a2 in 563 msec 2023-11-12T09:29:13,973 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:13,973 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:13,974 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:13,974 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781352975_default_test-1699781182417 2023-11-12T09:29:13,975 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781352975_default_test-1699781182417 2023-11-12T09:29:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=155 2023-11-12T09:29:14,385 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:14,390 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:14,390 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781352975_default_test-1699781182417 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781352975_default_test-1699781182417 2023-11-12T09:29:14,391 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:14,391 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 155 2023-11-12T09:29:14,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=155, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1699781352975_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } in 1.3990 sec 2023-11-12T09:29:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=155 2023-11-12T09:29:15,105 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781348644 2023-11-12T09:29:15,105 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:29:15,105 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-12T09:29:15,105 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781348644_test-1699781182417 2023-11-12T09:29:15,105 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781352975_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781348644/default/test-1699781182417/ 2023-11-12T09:29:15,105 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:29:15,128 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:29:15,128 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_741372034_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781348644/default/test-1699781182417, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781348644/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781352975_default_test-1699781182417 2023-11-12T09:29:15,128 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781352975_default_test-1699781182417. 2023-11-12T09:29:15,135 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781352975_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781348644/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781352975_default_test-1699781182417 2023-11-12T09:29:15,170 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.6153846153846154, tune throughput to 80.77 MB/second 2023-11-12T09:29:15,547 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:15,547 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:15,548 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:15,548 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:16,507 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-7187427314399973091.jar 2023-11-12T09:29:16,507 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:16,508 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:16,508 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:16,508 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:16,509 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:16,509 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:16,509 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:16,510 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:16,510 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:16,511 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:16,511 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:29:16,512 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:29:16,512 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:29:16,513 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:29:16,513 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:29:16,513 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:29:16,514 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:29:16,514 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:29:16,514 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:29:16,515 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:29:16,515 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:29:16,516 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:16,516 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:16,517 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:16,517 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:16,517 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:16,518 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:16,518 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:16,559 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781352975_default_test-1699781182417' hfile list 2023-11-12T09:29:16,561 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-12T09:29:16,629 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781356596/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:29:16,629 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781356596/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:29:16,690 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:29:16,736 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_ output=hdfs://localhost:34955/backupUT/backup_1699781348644/default/test-1699781182417/archive/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_ 2023-11-12T09:29:16,736 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 4.160M/sec 2023-11-12T09:29:17,661 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:29:17,662 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:29:17,667 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781352975_default_test-1699781182417 2023-11-12T09:29:17,667 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781352975_default_test-1699781182417 finished. 2023-11-12T09:29:17,667 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): test-1699781182417 2023-11-12T09:29:17,672 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781348644 2023-11-12T09:29:17,673 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:29:18,077 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781348644/.backup.manifest 2023-11-12T09:29:18,078 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(358): Backup backup_1699781348644 finished: type=FULL,tablelist=test-1699781182417,targetRootDir=hdfs://localhost:34955/backupUT,startts=1699781352785,completets=1699781357672,bytescopied=0 2023-11-12T09:29:18,078 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:29:18,078 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781352975_default_test-1699781182417 2023-11-12T09:29:18,078 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781352975_default_test-1699781182417" 2023-11-12T09:29:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781352975_default_test-1699781182417 2023-11-12T09:29:18,081 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781352975_default_test-1699781182417 for backup backup_1699781348644 succeeded. 2023-11-12T09:29:18,082 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:29:18,085 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:29:18,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:29:18,087 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:29:18,089 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:29:18,092 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(376): Backup backup_1699781348644 completed. 2023-11-12T09:29:18,093 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:18,103 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(265): backup complete 2023-11-12T09:29:18,103 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-11-12T09:29:18,104 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781348644/.backup.manifest 2023-11-12T09:29:18,106 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781348644/.backup.manifest 2023-11-12T09:29:18,107 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781348644/.backup.manifest 2023-11-12T09:29:18,109 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781348644/.backup.manifest 2023-11-12T09:29:18,109 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'faketable' to 'table1' from full backup image hdfs://localhost:34955/backupUT/backup_1699781348644/default/faketable 2023-11-12T09:29:18,128 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleDNE Thread=603 (was 605), OpenFileDescriptor=718 (was 708) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=162 (was 164), ProcessCount=171 (was 171), AvailableMemoryMB=5252 (was 5318) 2023-11-12T09:29:18,128 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-11-12T09:29:18,143 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleCommand Thread=603, OpenFileDescriptor=718, MaxFileDescriptor=60000, SystemLoadAverage=162, ProcessCount=171, AvailableMemoryMB=5251 2023-11-12T09:29:18,143 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-11-12T09:29:18,144 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(76): test full restore on a single table empty table: command-line 2023-11-12T09:29:18,150 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33208aae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:18,152 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:18,153 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:51000, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:18,154 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:18,156 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38576, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:18,169 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:29:18,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781358173 (current time:1699781358173). 2023-11-12T09:29:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:29:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@756b284a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:18,182 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:51008, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:18,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=157, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 157 2023-11-12T09:29:18,186 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-12T09:29:18,186 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:18,188 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:18,191 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-12T09:29:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-12T09:29:18,597 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:18,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0}] 2023-11-12T09:29:18,599 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:29:18,749 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:18,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=158 2023-11-12T09:29:18,750 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:29:18,750 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-12T09:29:18,754 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/7cdf52941fe140e58f05c2f79cd5fc96 is 169, key is trslm:hdfs://localhost:34955/backupUT\x00test-1699781182417/meta:log-roll-map/1699781357669/Put/seqid=0 2023-11-12T09:29:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-12T09:29:19,158 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/7cdf52941fe140e58f05c2f79cd5fc96 2023-11-12T09:29:19,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/97183c11ed3d45588d864e40a5005d80 is 452, key is session:backup_1699781348644/session:context/1699781358088/Put/seqid=0 2023-11-12T09:29:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-12T09:29:19,570 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/97183c11ed3d45588d864e40a5005d80 2023-11-12T09:29:19,575 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/7cdf52941fe140e58f05c2f79cd5fc96 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/7cdf52941fe140e58f05c2f79cd5fc96 2023-11-12T09:29:19,579 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/7cdf52941fe140e58f05c2f79cd5fc96, entries=4, sequenceid=127, filesize=5.5 K 2023-11-12T09:29:19,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/97183c11ed3d45588d864e40a5005d80 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/97183c11ed3d45588d864e40a5005d80 2023-11-12T09:29:19,585 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/97183c11ed3d45588d864e40a5005d80, entries=2, sequenceid=127, filesize=5.4 K 2023-11-12T09:29:19,586 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 836ms, sequenceid=127, compaction requested=true 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(2520): Flush status journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. for snapshot_backup_system completed. 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0703c67cb50e4c9eb5a47d31720493c9, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/2fb3277523ad45e0a39359b7d56e92cc, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/5d7a351cc7434a339a200f712867b7ff, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/7cdf52941fe140e58f05c2f79cd5fc96] hfiles 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (1/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (2/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (3/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (4/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (5/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (6/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (7/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (8/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0703c67cb50e4c9eb5a47d31720493c9 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (9/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/2fb3277523ad45e0a39359b7d56e92cc for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (10/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/5d7a351cc7434a339a200f712867b7ff for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (11/11): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/7cdf52941fe140e58f05c2f79cd5fc96 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/11d8abd27f484bf1b60368d701936880, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/3ab1f4f4f59e4ddfbd921088b0ee4cb2, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/1615bcf1a8bf41eab0b3ea9ba058fb43, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/97183c11ed3d45588d864e40a5005d80] hfiles 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (1/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (2/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (3/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (4/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c for snapshot=snapshot_backup_system 2023-11-12T09:29:19,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (5/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (6/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (7/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (8/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (9/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/11d8abd27f484bf1b60368d701936880 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (10/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/3ab1f4f4f59e4ddfbd921088b0ee4cb2 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (11/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/1615bcf1a8bf41eab0b3ea9ba058fb43 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (12/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/97183c11ed3d45588d864e40a5005d80 for snapshot=snapshot_backup_system 2023-11-12T09:29:19,992 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:29:19,992 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=158 2023-11-12T09:29:19,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=158 2023-11-12T09:29:19,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 606338a328e82acae51355d485ed81f0 2023-11-12T09:29:19,994 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:29:19,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=158, resume processing ppid=157 2023-11-12T09:29:19,995 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:19,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=158, ppid=157, state=SUCCESS; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 in 1.3970 sec 2023-11-12T09:29:19,996 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:19,996 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:19,996 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:29:19,997 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:29:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-12T09:29:20,408 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:20,420 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:20,421 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:29:20,422 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:20,422 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 157 2023-11-12T09:29:20,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=157, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2390 sec 2023-11-12T09:29:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-12T09:29:22,293 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(115): Backup backup_1699781358150 started at 1699781362293. 2023-11-12T09:29:22,294 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(119): Backup session backup_1699781358150 has been started. 2023-11-12T09:29:22,298 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:29:22,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:29:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:29:22,299 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:29:22,299 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:29:22,300 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:29:22,300 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:29:22,301 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:22,302 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:22,302 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:22,302 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:22,303 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:29:22,303 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:22,303 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:29:22,303 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:22,303 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:29:22,304 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:29:22,304 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:29:22,304 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:29:22,304 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:29:22,305 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:29:22,305 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:29:22,305 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:29:22,307 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:22,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:22,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:22,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:22,307 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:29:22,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:22,308 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:22,308 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:22,308 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:22,308 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:22,309 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:29:22,309 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@8504478[Count = 0] remaining members to acquire global barrier 2023-11-12T09:29:22,309 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:29:22,309 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:22,312 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:22,312 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:22,312 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:29:22,312 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:29:22,312 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:29:22,312 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:22,312 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781352804 highest: 1699781352804 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:22,312 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781352804) roll requested 2023-11-12T09:29:22,329 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781362313, exclude list is [], retry=0 2023-11-12T09:29:22,332 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:29:22,334 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781352804 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781362313 2023-11-12T09:29:22,334 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:29:22,334 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781352804 is not closed yet, will try archiving it next time 2023-11-12T09:29:22,334 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781331442 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781331442 2023-11-12T09:29:22,334 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781352824) roll requested 2023-11-12T09:29:22,346 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781362334.meta, exclude list is [], retry=0 2023-11-12T09:29:22,349 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:29:22,350 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781352824.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781362334.meta 2023-11-12T09:29:22,350 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:29:22,350 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781352824.meta is not closed yet, will try archiving it next time 2023-11-12T09:29:22,352 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781352824.meta to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781352824.meta 2023-11-12T09:29:22,353 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-11-12T09:29:22,353 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781362313 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:22,359 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:29:22,359 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36287,1699781172353' in zk 2023-11-12T09:29:22,363 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:29:22,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:22,363 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:22,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:22,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:22,363 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:29:22,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:22,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:22,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:22,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:22,366 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:22,366 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:22,366 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:22,367 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36287,1699781172353': 2023-11-12T09:29:22,367 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:29:22,367 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:29:22,367 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:29:22,367 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:29:22,367 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:29:22,367 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:29:22,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:22,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:22,369 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:29:22,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:22,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:22,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:22,369 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:22,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:22,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:22,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:22,370 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:22,370 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:22,370 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:22,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:22,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:22,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:22,375 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:22,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:22,375 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:22,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:29:22,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:29:22,376 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:22,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:29:22,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:29:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781362480 (current time:1699781362480). 2023-11-12T09:29:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781362479_default_test-1699781182417 VERSION not specified, setting to 2 2023-11-12T09:29:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dabf5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:22,489 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:51022, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=159, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } 2023-11-12T09:29:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 159 2023-11-12T09:29:22,492 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=159 2023-11-12T09:29:22,493 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:22,496 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781362479_default_test-1699781182417 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:22,500 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=159 2023-11-12T09:29:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=159 2023-11-12T09:29:22,909 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:22,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure 3ca9201e34d84739ee34b92f9445e4a2}] 2023-11-12T09:29:22,911 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:29:23,062 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:23,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=160 2023-11-12T09:29:23,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:29:23,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.HRegion(2520): Flush status journal for 3ca9201e34d84739ee34b92f9445e4a2: 2023-11-12T09:29:23,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. for snapshot_1699781362479_default_test-1699781182417 completed. 2023-11-12T09:29:23,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(240): Storing 'test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2.' region-info for snapshot=snapshot_1699781362479_default_test-1699781182417 2023-11-12T09:29:23,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:23,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_] hfiles 2023-11-12T09:29:23,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_ for snapshot=snapshot_1699781362479_default_test-1699781182417 2023-11-12T09:29:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=159 2023-11-12T09:29:23,469 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:29:23,469 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=160 2023-11-12T09:29:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=160 2023-11-12T09:29:23,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781362479_default_test-1699781182417 on region 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:29:23,471 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:29:23,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=160, resume processing ppid=159 2023-11-12T09:29:23,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=160, ppid=159, state=SUCCESS; SnapshotRegionProcedure 3ca9201e34d84739ee34b92f9445e4a2 in 562 msec 2023-11-12T09:29:23,472 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:23,473 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:23,473 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:23,473 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781362479_default_test-1699781182417 2023-11-12T09:29:23,474 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781362479_default_test-1699781182417 2023-11-12T09:29:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=159 2023-11-12T09:29:23,883 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:23,888 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:23,888 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781362479_default_test-1699781182417 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781362479_default_test-1699781182417 2023-11-12T09:29:23,889 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:23,889 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 }, snapshot procedure id = 159 2023-11-12T09:29:23,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=159, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1699781362479_default_test-1699781182417 table=test-1699781182417 type=FLUSH ttl=0 } in 1.3990 sec 2023-11-12T09:29:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=159 2023-11-12T09:29:24,597 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781358150 2023-11-12T09:29:24,597 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:29:24,597 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-12T09:29:24,597 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781358150_test-1699781182417 2023-11-12T09:29:24,597 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781362479_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417/ 2023-11-12T09:29:24,597 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:29:24,620 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:29:24,620 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_1053790193_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781362479_default_test-1699781182417 2023-11-12T09:29:24,620 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781362479_default_test-1699781182417. 2023-11-12T09:29:24,627 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781362479_default_test-1699781182417 to hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417/.hbase-snapshot/.tmp/snapshot_1699781362479_default_test-1699781182417 2023-11-12T09:29:25,040 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:25,040 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:25,040 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:25,041 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:26,006 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-1299328872054552476.jar 2023-11-12T09:29:26,007 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:26,007 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:26,008 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:26,008 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:26,008 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:26,009 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:26,009 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:26,009 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:26,009 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:26,010 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:26,011 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:29:26,011 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:29:26,011 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:29:26,012 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:29:26,012 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:29:26,013 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:29:26,013 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:29:26,013 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:29:26,013 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:29:26,014 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:29:26,014 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:29:26,015 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:26,015 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:26,016 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:26,016 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:26,017 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:26,017 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:26,018 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:26,059 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781362479_default_test-1699781182417' hfile list 2023-11-12T09:29:26,061 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-12T09:29:26,133 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781366097/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:29:26,133 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781366097/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:29:26,187 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:29:26,210 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_ output=hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417/archive/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_ 2023-11-12T09:29:26,210 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-11-12T09:29:27,165 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:29:27,166 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:29:27,172 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781362479_default_test-1699781182417 2023-11-12T09:29:27,172 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781362479_default_test-1699781182417 finished. 2023-11-12T09:29:27,172 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): test-1699781182417 2023-11-12T09:29:27,177 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781358150 2023-11-12T09:29:27,177 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:29:27,582 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781358150/.backup.manifest 2023-11-12T09:29:27,582 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(358): Backup backup_1699781358150 finished: type=FULL,tablelist=test-1699781182417,targetRootDir=hdfs://localhost:34955/backupUT,startts=1699781362293,completets=1699781367177,bytescopied=0 2023-11-12T09:29:27,582 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:29:27,582 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781362479_default_test-1699781182417 2023-11-12T09:29:27,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781362479_default_test-1699781182417" 2023-11-12T09:29:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781362479_default_test-1699781182417 2023-11-12T09:29:27,586 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781362479_default_test-1699781182417 for backup backup_1699781358150 succeeded. 2023-11-12T09:29:27,586 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:29:27,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:29:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:29:27,592 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:29:27,594 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:29:27,599 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(376): Backup backup_1699781358150 completed. 2023-11-12T09:29:27,599 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:27,599 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(80): backup complete 2023-11-12T09:29:27,615 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31014b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:27,617 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:27,618 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45486, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:27,619 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-11-12T09:29:27,620 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781358150/.backup.manifest 2023-11-12T09:29:27,622 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781358150/.backup.manifest 2023-11-12T09:29:27,622 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781358150/.backup.manifest 2023-11-12T09:29:27,624 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781358150/.backup.manifest 2023-11-12T09:29:27,624 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'test-1699781182417' to 'table1' from full backup image hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417 2023-11-12T09:29:27,630 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(487): Creating target table 'table1' 2023-11-12T09:29:27,630 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417/archive/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:29:27,631 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417/archive/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f in region [hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417/archive/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2] 2023-11-12T09:29:27,634 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417/archive/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_ first=row0 last=row98 2023-11-12T09:29:27,635 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:27,636 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39214, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:27,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:29:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=161, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-11-12T09:29:27,641 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:29:27,641 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 161 2023-11-12T09:29:27,642 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:29:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=161 2023-11-12T09:29:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=161 2023-11-12T09:29:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=161 2023-11-12T09:29:28,049 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 93c5ba59e3a56dfa1b379b332a7e6f1f, NAME => 'table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:29:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=161 2023-11-12T09:29:28,455 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:29:28,455 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing 93c5ba59e3a56dfa1b379b332a7e6f1f, disabling compactions & flushes 2023-11-12T09:29:28,455 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f. 2023-11-12T09:29:28,455 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f. 2023-11-12T09:29:28,455 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f. after waiting 0 ms 2023-11-12T09:29:28,455 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f. 2023-11-12T09:29:28,455 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f. 2023-11-12T09:29:28,455 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for 93c5ba59e3a56dfa1b379b332a7e6f1f: 2023-11-12T09:29:28,456 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:29:28,456 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1699781368456"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781368456"}]},"ts":"1699781368456"} 2023-11-12T09:29:28,457 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:29:28,458 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:29:28,458 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781368458"}]},"ts":"1699781368458"} 2023-11-12T09:29:28,459 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-11-12T09:29:28,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=93c5ba59e3a56dfa1b379b332a7e6f1f, ASSIGN}] 2023-11-12T09:29:28,466 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=93c5ba59e3a56dfa1b379b332a7e6f1f, ASSIGN 2023-11-12T09:29:28,466 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=93c5ba59e3a56dfa1b379b332a7e6f1f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,36287,1699781172353; forceNewPlan=false, retain=false 2023-11-12T09:29:28,620 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=162 updating hbase:meta row=93c5ba59e3a56dfa1b379b332a7e6f1f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:28,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE; OpenRegionProcedure 93c5ba59e3a56dfa1b379b332a7e6f1f, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:29:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=161 2023-11-12T09:29:28,776 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:28,779 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(130): Open table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f. 2023-11-12T09:29:28,779 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7254): Opening region: {ENCODED => 93c5ba59e3a56dfa1b379b332a7e6f1f, NAME => 'table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:29:28,779 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:29:28,779 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 93c5ba59e3a56dfa1b379b332a7e6f1f 2023-11-12T09:29:28,779 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(888): Instantiated table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:29:28,780 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7296): checking encryption for 93c5ba59e3a56dfa1b379b332a7e6f1f 2023-11-12T09:29:28,780 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7299): checking classloading for 93c5ba59e3a56dfa1b379b332a7e6f1f 2023-11-12T09:29:28,781 INFO [StoreOpener-93c5ba59e3a56dfa1b379b332a7e6f1f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 93c5ba59e3a56dfa1b379b332a7e6f1f 2023-11-12T09:29:28,782 INFO [StoreOpener-93c5ba59e3a56dfa1b379b332a7e6f1f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93c5ba59e3a56dfa1b379b332a7e6f1f columnFamilyName f 2023-11-12T09:29:28,782 DEBUG [StoreOpener-93c5ba59e3a56dfa1b379b332a7e6f1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:29:28,782 INFO [StoreOpener-93c5ba59e3a56dfa1b379b332a7e6f1f-1 {}] regionserver.HStore(324): Store=93c5ba59e3a56dfa1b379b332a7e6f1f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:29:28,783 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f 2023-11-12T09:29:28,783 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f 2023-11-12T09:29:28,785 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1076): writing seq id for 93c5ba59e3a56dfa1b379b332a7e6f1f 2023-11-12T09:29:28,787 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:29:28,787 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1093): Opened 93c5ba59e3a56dfa1b379b332a7e6f1f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12049209120, jitterRate=0.12217004597187042}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:29:28,788 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(992): Region open journal for 93c5ba59e3a56dfa1b379b332a7e6f1f: 2023-11-12T09:29:28,788 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f., pid=163, masterSystemTime=1699781368775 2023-11-12T09:29:28,789 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f. 2023-11-12T09:29:28,789 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(158): Opened table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f. 2023-11-12T09:29:28,790 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=162 updating hbase:meta row=93c5ba59e3a56dfa1b379b332a7e6f1f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:28,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=163, resume processing ppid=162 2023-11-12T09:29:28,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=163, ppid=162, state=SUCCESS; OpenRegionProcedure 93c5ba59e3a56dfa1b379b332a7e6f1f, server=jenkins-hbase3.apache.org,36287,1699781172353 in 170 msec 2023-11-12T09:29:28,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=162, resume processing ppid=161 2023-11-12T09:29:28,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=162, ppid=161, state=SUCCESS; TransitRegionStateProcedure table=table1, region=93c5ba59e3a56dfa1b379b332a7e6f1f, ASSIGN in 328 msec 2023-11-12T09:29:28,794 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:29:28,794 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781368794"}]},"ts":"1699781368794"} 2023-11-12T09:29:28,795 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-11-12T09:29:28,797 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:29:28,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=161, state=SUCCESS; CreateTableProcedure table=table1 in 1.1600 sec 2023-11-12T09:29:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=161 2023-11-12T09:29:29,753 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417/archive/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2 from hbase tables test-1699781182417 to tables table1 2023-11-12T09:29:29,753 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1699781182417 into table1 2023-11-12T09:29:29,755 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781369753 from hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417/archive/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:29:29,762 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea4aff9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:29,764 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:29,766 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39224, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:29,768 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:29:29,768 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-11-12T09:29:29,772 DEBUG [hconnection-0x2b910f1-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:29,773 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45492, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:29,774 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-11-12T09:29:29,774 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-12T09:29:29,774 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/partitions_2a989189-65ec-42ef-96a0-bf79fb389d7a 2023-11-12T09:29:30,184 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:30,185 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:30,186 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:30,186 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:30,347 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-11-12T09:29:30,347 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:29:31,182 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-6086378010918988957.jar 2023-11-12T09:29:31,183 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,183 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,183 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,184 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,184 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,184 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,185 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,185 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,185 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,186 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:31,187 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:29:31,187 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:29:31,187 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:29:31,188 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:29:31,188 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:29:31,188 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:29:31,189 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:29:31,189 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:29:31,189 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:29:31,190 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:29:31,190 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:29:31,191 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,191 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,191 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,192 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,192 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,192 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:31,193 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:31,193 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-11-12T09:29:31,193 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:31,193 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-12T09:29:31,193 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-12T09:29:31,194 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:31,206 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-12T09:29:31,280 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781371246/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:29:31,280 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781371246/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:29:31,350 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417/archive/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/f/4ff211ab617640bba6d3ee98e34ed017_SeqId_4_ 2023-11-12T09:29:31,354 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-12T09:29:31,373 DEBUG [pool-536-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4132cb1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:31,375 DEBUG [hconnection-0x646bd845-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:31,376 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45508, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:31,380 DEBUG [pool-536-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:31,380 DEBUG [pool-536-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-12T09:29:31,380 DEBUG [pool-536-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:29:31,384 DEBUG [pool-536-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781369753/_temporary/0/_temporary/attempt_local1961771555_0025_r_000000_0/f/4bee7d3d5f4e4d079735f2b71eb67bcd is 37, key is row10/f:q1/1699781185078/Put/seqid=0 2023-11-12T09:29:31,385 WARN [Thread-2350 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:36287] Specified favored nodes: [jenkins-hbase3.apache.org:36287] 2023-11-12T09:29:32,324 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781369753 2023-11-12T09:29:32,329 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63e2bb1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:32,331 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:32,332 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45512, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:32,337 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781369753/_SUCCESS 2023-11-12T09:29:32,341 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:32,342 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39228, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:32,350 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781369753/f/4bee7d3d5f4e4d079735f2b71eb67bcd first=Optional[row0] last=Optional[row98] 2023-11-12T09:29:32,351 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f., hostname=jenkins-hbase3.apache.org,36287,1699781172353, seqNum=2 for row with hfile group [{f,hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781369753/f/4bee7d3d5f4e4d079735f2b71eb67bcd}] 2023-11-12T09:29:32,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781369753/f/4bee7d3d5f4e4d079735f2b71eb67bcd for inclusion in 93c5ba59e3a56dfa1b379b332a7e6f1f/f 2023-11-12T09:29:32,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-12T09:29:32,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-12T09:29:32,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegion(2520): Flush status journal for 93c5ba59e3a56dfa1b379b332a7e6f1f: 2023-11-12T09:29:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/hbase-staging/bulk_output-default-table1-1699781369753/f/4bee7d3d5f4e4d079735f2b71eb67bcd to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__table1__96faslk8sn76u1fhdia1fdcl9vnf0etouamgk27ol67b86jh3ki771ia3i1l9a8v/f/4bee7d3d5f4e4d079735f2b71eb67bcd 2023-11-12T09:29:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__table1__96faslk8sn76u1fhdia1fdcl9vnf0etouamgk27ol67b86jh3ki771ia3i1l9a8v/f/4bee7d3d5f4e4d079735f2b71eb67bcd as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f/f/c2a54da8d3494eddb8171d7fc469e11f_SeqId_4_ 2023-11-12T09:29:32,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55899da2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:32,371 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39236, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:32,375 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45520, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:32,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__table1__96faslk8sn76u1fhdia1fdcl9vnf0etouamgk27ol67b86jh3ki771ia3i1l9a8v/f/4bee7d3d5f4e4d079735f2b71eb67bcd into 93c5ba59e3a56dfa1b379b332a7e6f1f/f as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f/f/c2a54da8d3494eddb8171d7fc469e11f_SeqId_4_ - updating store file list. 2023-11-12T09:29:32,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f/f/c2a54da8d3494eddb8171d7fc469e11f_SeqId_4_ into 93c5ba59e3a56dfa1b379b332a7e6f1f/f 2023-11-12T09:29:32,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__table1__96faslk8sn76u1fhdia1fdcl9vnf0etouamgk27ol67b86jh3ki771ia3i1l9a8v/f/4bee7d3d5f4e4d079735f2b71eb67bcd into 93c5ba59e3a56dfa1b379b332a7e6f1f/f (new location: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f/f/c2a54da8d3494eddb8171d7fc469e11f_SeqId_4_) 2023-11-12T09:29:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/staging/jenkins__table1__96faslk8sn76u1fhdia1fdcl9vnf0etouamgk27ol67b86jh3ki771ia3i1l9a8v/f/4bee7d3d5f4e4d079735f2b71eb67bcd 2023-11-12T09:29:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d38142, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:32,400 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39242, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:29:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:32,404 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45524, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-12T09:29:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:32,416 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:32,416 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-12T09:29:32,416 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-12T09:29:32,416 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(233): Backup: backup_1699781358150 hdfs://localhost:34955/backupUT/backup_1699781358150/default/test-1699781182417/ 2023-11-12T09:29:32,416 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-12T09:29:32,416 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:32,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable table1 2023-11-12T09:29:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-11-12T09:29:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=164 2023-11-12T09:29:32,422 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781372421"}]},"ts":"1699781372421"} 2023-11-12T09:29:32,422 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-11-12T09:29:32,425 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-11-12T09:29:32,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=93c5ba59e3a56dfa1b379b332a7e6f1f, UNASSIGN}] 2023-11-12T09:29:32,426 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=93c5ba59e3a56dfa1b379b332a7e6f1f, UNASSIGN 2023-11-12T09:29:32,427 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=93c5ba59e3a56dfa1b379b332a7e6f1f, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:32,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE; CloseRegionProcedure 93c5ba59e3a56dfa1b379b332a7e6f1f, server=jenkins-hbase3.apache.org,36287,1699781172353}] 2023-11-12T09:29:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=164 2023-11-12T09:29:32,579 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:32,580 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(119): Close 93c5ba59e3a56dfa1b379b332a7e6f1f 2023-11-12T09:29:32,580 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1668): Closing 93c5ba59e3a56dfa1b379b332a7e6f1f, disabling compactions & flushes 2023-11-12T09:29:32,580 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1690): Closing region table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f. 2023-11-12T09:29:32,580 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f. 2023-11-12T09:29:32,580 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1778): Acquired close lock on table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f. after waiting 0 ms 2023-11-12T09:29:32,580 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1788): Updates disabled for region table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f. 2023-11-12T09:29:32,586 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-12T09:29:32,586 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:32,586 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1909): Closed table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f. 2023-11-12T09:29:32,586 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1622): Region close journal for 93c5ba59e3a56dfa1b379b332a7e6f1f: 2023-11-12T09:29:32,588 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(163): Closed 93c5ba59e3a56dfa1b379b332a7e6f1f 2023-11-12T09:29:32,588 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=93c5ba59e3a56dfa1b379b332a7e6f1f, regionState=CLOSED 2023-11-12T09:29:32,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=166, resume processing ppid=165 2023-11-12T09:29:32,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=166, ppid=165, state=SUCCESS; CloseRegionProcedure 93c5ba59e3a56dfa1b379b332a7e6f1f, server=jenkins-hbase3.apache.org,36287,1699781172353 in 161 msec 2023-11-12T09:29:32,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=165, resume processing ppid=164 2023-11-12T09:29:32,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=165, ppid=164, state=SUCCESS; TransitRegionStateProcedure table=table1, region=93c5ba59e3a56dfa1b379b332a7e6f1f, UNASSIGN in 165 msec 2023-11-12T09:29:32,592 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781372592"}]},"ts":"1699781372592"} 2023-11-12T09:29:32,593 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-11-12T09:29:32,595 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-11-12T09:29:32,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=164, state=SUCCESS; DisableTableProcedure table=table1 in 176 msec 2023-11-12T09:29:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=164 2023-11-12T09:29:32,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster$5(2449): Client=jenkins//172.31.12.81 delete table1 2023-11-12T09:29:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=167, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-11-12T09:29:32,727 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=167, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:29:32,728 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=167, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:29:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=167 2023-11-12T09:29:32,730 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f 2023-11-12T09:29:32,731 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f/f, FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f/recovered.edits] 2023-11-12T09:29:32,736 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f/f/c2a54da8d3494eddb8171d7fc469e11f_SeqId_4_ to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f/f/c2a54da8d3494eddb8171d7fc469e11f_SeqId_4_ 2023-11-12T09:29:32,739 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f/recovered.edits/6.seqid to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/archive/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f/recovered.edits/6.seqid 2023-11-12T09:29:32,739 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/table1/93c5ba59e3a56dfa1b379b332a7e6f1f 2023-11-12T09:29:32,740 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-11-12T09:29:32,742 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=167, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:29:32,744 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-11-12T09:29:32,745 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-11-12T09:29:32,747 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=167, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:29:32,747 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-11-12T09:29:32,747 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699781372747"}]},"ts":"9223372036854775807"} 2023-11-12T09:29:32,748 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:29:32,748 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 93c5ba59e3a56dfa1b379b332a7e6f1f, NAME => 'table1,,1699781367637.93c5ba59e3a56dfa1b379b332a7e6f1f.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:29:32,748 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-11-12T09:29:32,749 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1699781372748"}]},"ts":"9223372036854775807"} 2023-11-12T09:29:32,750 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-11-12T09:29:32,752 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=167, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-12T09:29:32,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=167, state=SUCCESS; DeleteTableProcedure table=table1 in 28 msec 2023-11-12T09:29:32,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=167 2023-11-12T09:29:32,849 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleCommand Thread=605 (was 603) - Thread LEAK? -, OpenFileDescriptor=713 (was 718), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=126 (was 162), ProcessCount=171 (was 171), AvailableMemoryMB=5196 (was 5251) 2023-11-12T09:29:32,849 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-12T09:29:32,868 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleDNE Thread=605, OpenFileDescriptor=713, MaxFileDescriptor=60000, SystemLoadAverage=126, ProcessCount=171, AvailableMemoryMB=5196 2023-11-12T09:29:32,868 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-12T09:29:32,869 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] backup.TestFullRestore(302): test restore fails on multiple tables that do not exist 2023-11-12T09:29:32,875 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70c35499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:32,878 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:32,880 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45526, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:32,882 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:29:32,884 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:39250, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:29:32,893 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:29:32,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781372898 (current time:1699781372898). 2023-11-12T09:29:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:29:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a31d99a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:32,908 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45528, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:32,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:32,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=168, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:29:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 168 2023-11-12T09:29:32,913 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-12T09:29:32,914 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:32,916 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:32,919 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-12T09:29:33,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-12T09:29:33,324 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:33,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0}] 2023-11-12T09:29:33,326 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:29:33,477 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:33,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=169 2023-11-12T09:29:33,478 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:29:33,478 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-12T09:29:33,481 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/a84169d4ca4a4bb3842f3ec43aa6c258 is 169, key is trslm:hdfs://localhost:34955/backupUT\x00test-1699781182417/meta:log-roll-map/1699781367174/Put/seqid=0 2023-11-12T09:29:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-12T09:29:33,886 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/a84169d4ca4a4bb3842f3ec43aa6c258 2023-11-12T09:29:33,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/7fd5406423494a609fd4c40195f5d1f9 is 452, key is session:backup_1699781358150/session:context/1699781367593/Put/seqid=0 2023-11-12T09:29:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-12T09:29:34,297 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/7fd5406423494a609fd4c40195f5d1f9 2023-11-12T09:29:34,302 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/a84169d4ca4a4bb3842f3ec43aa6c258 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/a84169d4ca4a4bb3842f3ec43aa6c258 2023-11-12T09:29:34,306 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/a84169d4ca4a4bb3842f3ec43aa6c258, entries=4, sequenceid=138, filesize=5.5 K 2023-11-12T09:29:34,307 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/7fd5406423494a609fd4c40195f5d1f9 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/7fd5406423494a609fd4c40195f5d1f9 2023-11-12T09:29:34,311 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/7fd5406423494a609fd4c40195f5d1f9, entries=2, sequenceid=138, filesize=5.4 K 2023-11-12T09:29:34,312 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 834ms, sequenceid=138, compaction requested=true 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(2520): Flush status journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. for snapshot_backup_system completed. 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781191829.606338a328e82acae51355d485ed81f0.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0703c67cb50e4c9eb5a47d31720493c9, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/2fb3277523ad45e0a39359b7d56e92cc, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/5d7a351cc7434a339a200f712867b7ff, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/7cdf52941fe140e58f05c2f79cd5fc96, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/a84169d4ca4a4bb3842f3ec43aa6c258] hfiles 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (1/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/6fb01fef96694886ae3b55c3915531db for snapshot=snapshot_backup_system 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (2/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0d301e1aaa1e43b798e7fbad9ead0c81 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (3/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/17f27685c473467aa6033e9335fdd0bc for snapshot=snapshot_backup_system 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (4/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/977a0036f3404b628b014de901c01bfd for snapshot=snapshot_backup_system 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (5/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/3b495e5a835a4c48b378e5ad24d02c00 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (6/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/f82d4a97dcba4ae5b07cff1d7ec7effa for snapshot=snapshot_backup_system 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (7/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/84fb173346f74b8e9ff8d878b5974b87 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (8/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/0703c67cb50e4c9eb5a47d31720493c9 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (9/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/2fb3277523ad45e0a39359b7d56e92cc for snapshot=snapshot_backup_system 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (10/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/5d7a351cc7434a339a200f712867b7ff for snapshot=snapshot_backup_system 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (11/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/7cdf52941fe140e58f05c2f79cd5fc96 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (12/12): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/a84169d4ca4a4bb3842f3ec43aa6c258 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/11d8abd27f484bf1b60368d701936880, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/3ab1f4f4f59e4ddfbd921088b0ee4cb2, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/1615bcf1a8bf41eab0b3ea9ba058fb43, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/97183c11ed3d45588d864e40a5005d80, hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/7fd5406423494a609fd4c40195f5d1f9] hfiles 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (1/13): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/033eb5dea61d4d04a65e5f0f5582e1e8 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (2/13): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/d247c51ad9b84f17a7cb32cc384f5c0b for snapshot=snapshot_backup_system 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (3/13): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/e010cb2a3e0b4113b2209bc702d697bd for snapshot=snapshot_backup_system 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (4/13): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/6e038692cf954f1084e0b11d4b62020c for snapshot=snapshot_backup_system 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (5/13): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/adb71489e30e4e53a48f0bbd3bb4f3b5 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (6/13): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/40236c1a3b484da8a8d88269f03914e8 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (7/13): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/0c9e5bc5cf094ea981f2054c8ef12fc3 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (8/13): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/4c19f00d68984382ac24c49b4f6d0244 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (9/13): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/11d8abd27f484bf1b60368d701936880 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (10/13): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/3ab1f4f4f59e4ddfbd921088b0ee4cb2 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (11/13): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/1615bcf1a8bf41eab0b3ea9ba058fb43 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (12/13): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/97183c11ed3d45588d864e40a5005d80 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,313 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (13/13): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/7fd5406423494a609fd4c40195f5d1f9 for snapshot=snapshot_backup_system 2023-11-12T09:29:34,318 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:29:34,718 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:29:34,718 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=169 2023-11-12T09:29:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=169 2023-11-12T09:29:34,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 606338a328e82acae51355d485ed81f0 2023-11-12T09:29:34,720 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 2023-11-12T09:29:34,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=169, resume processing ppid=168 2023-11-12T09:29:34,721 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:34,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=169, ppid=168, state=SUCCESS; SnapshotRegionProcedure 606338a328e82acae51355d485ed81f0 in 1.3950 sec 2023-11-12T09:29:34,722 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:34,722 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:34,722 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:29:34,723 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:29:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-12T09:29:35,134 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:35,147 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:35,147 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:29:35,148 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:35,148 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 168 2023-11-12T09:29:35,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=168, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2380 sec 2023-11-12T09:29:37,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-12T09:29:37,018 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(115): Backup backup_1699781372875 started at 1699781377018. 2023-11-12T09:29:37,020 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(119): Backup session backup_1699781372875 has been started. 2023-11-12T09:29:37,024 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:29:37,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:29:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:29:37,025 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:29:37,025 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:29:37,026 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:29:37,026 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:29:37,026 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:37,028 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:37,028 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:37,028 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:37,028 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:29:37,028 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:37,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:29:37,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:29:37,029 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:29:37,029 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:29:37,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:29:37,029 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:29:37,029 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:29:37,030 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:29:37,030 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:29:37,030 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:29:37,032 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:37,032 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:37,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:37,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:37,033 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:29:37,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:37,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:37,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:37,034 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:37,034 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:37,034 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,36287,1699781172353' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:29:37,034 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@55a5ad71[Count = 0] remaining members to acquire global barrier 2023-11-12T09:29:37,034 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:29:37,034 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:37,035 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:37,035 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:29:37,035 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:29:37,036 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:29:37,036 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:29:37,036 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:37,036 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781362313 highest: 1699781362313 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:37,036 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781362313) roll requested 2023-11-12T09:29:37,047 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781377036, exclude list is [], retry=0 2023-11-12T09:29:37,051 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:29:37,053 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781362313 with entries=13, filesize=3.13 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781377036 2023-11-12T09:29:37,053 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:29:37,053 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781362313 is not closed yet, will try archiving it next time 2023-11-12T09:29:37,053 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781362334) roll requested 2023-11-12T09:29:37,053 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781352804 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781352804 2023-11-12T09:29:37,055 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.1699781362313 not finished, retry = 0 2023-11-12T09:29:37,065 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781377053.meta, exclude list is [], retry=0 2023-11-12T09:29:37,068 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK] 2023-11-12T09:29:37,070 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781362334.meta with entries=12, filesize=2.74 KB; new WAL /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781377053.meta 2023-11-12T09:29:37,070 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36711,DS-32cb1215-5008-40d4-a569-9a97b9569b71,DISK]] 2023-11-12T09:29:37,070 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781362334.meta is not closed yet, will try archiving it next time 2023-11-12T09:29:37,072 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/WALs/jenkins-hbase3.apache.org,36287,1699781172353/jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta.1699781362334.meta not finished, retry = 0 2023-11-12T09:29:37,076 DEBUG [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:29:37,076 INFO [rs(jenkins-hbase3.apache.org,36287,1699781172353)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781377036 on jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:37,082 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:29:37,082 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,36287,1699781172353' in zk 2023-11-12T09:29:37,086 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:29:37,086 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:37,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:37,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:37,086 DEBUG [member: 'jenkins-hbase3.apache.org,36287,1699781172353' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:29:37,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:37,087 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:37,087 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:37,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:37,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:37,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:37,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:37,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:37,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,36287,1699781172353': 2023-11-12T09:29:37,090 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,36287,1699781172353' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:29:37,090 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:29:37,090 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:29:37,090 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:29:37,090 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:29:37,090 INFO [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:29:37,092 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:29:37,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:37,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:37,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:29:37,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:29:37,093 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:37,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:37,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:29:37,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:29:37,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:37,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:29:37,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:37,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:37,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:29:37,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:29:37,095 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:37,099 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:29:37,099 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:29:37,099 DEBUG [(jenkins-hbase3.apache.org,43639,1699781169211)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:29:37,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:29:37,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:29:37,100 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:29:37,100 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:29:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:29:37,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } 2023-11-12T09:29:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781377203 (current time:1699781377203). 2023-11-12T09:29:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:37,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781377203_ns2_test-16997811824171 VERSION not specified, setting to 2 2023-11-12T09:29:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37fdd62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:37,214 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49606, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=170, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } 2023-11-12T09:29:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 }, snapshot procedure id = 170 2023-11-12T09:29:37,217 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=170 2023-11-12T09:29:37,218 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:37,220 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781377203_ns2_test-16997811824171 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:37,223 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=170 2023-11-12T09:29:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=170 2023-11-12T09:29:37,628 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:37,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure d347587386821efa96e22f590d908cde}] 2023-11-12T09:29:37,630 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure d347587386821efa96e22f590d908cde 2023-11-12T09:29:37,781 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:37,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=171 2023-11-12T09:29:37,781 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:29:37,781 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.HRegion(2520): Flush status journal for d347587386821efa96e22f590d908cde: 2023-11-12T09:29:37,781 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. for snapshot_1699781377203_ns2_test-16997811824171 completed. 2023-11-12T09:29:37,782 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde.' region-info for snapshot=snapshot_1699781377203_ns2_test-16997811824171 2023-11-12T09:29:37,782 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:37,782 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_] hfiles 2023-11-12T09:29:37,782 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_ for snapshot=snapshot_1699781377203_ns2_test-16997811824171 2023-11-12T09:29:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=170 2023-11-12T09:29:38,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:29:38,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=171 2023-11-12T09:29:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=171 2023-11-12T09:29:38,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781377203_ns2_test-16997811824171 on region d347587386821efa96e22f590d908cde 2023-11-12T09:29:38,189 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure d347587386821efa96e22f590d908cde 2023-11-12T09:29:38,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=171, resume processing ppid=170 2023-11-12T09:29:38,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=171, ppid=170, state=SUCCESS; SnapshotRegionProcedure d347587386821efa96e22f590d908cde in 561 msec 2023-11-12T09:29:38,191 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:38,191 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:38,191 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:38,191 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781377203_ns2_test-16997811824171 2023-11-12T09:29:38,192 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781377203_ns2_test-16997811824171 2023-11-12T09:29:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=170 2023-11-12T09:29:38,600 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:38,605 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:38,605 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781377203_ns2_test-16997811824171 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781377203_ns2_test-16997811824171 2023-11-12T09:29:38,606 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:38,606 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 }, snapshot procedure id = 170 2023-11-12T09:29:38,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=170, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1699781377203_ns2_test-16997811824171 table=ns2:test-16997811824171 type=FLUSH ttl=0 } in 1.3910 sec 2023-11-12T09:29:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=170 2023-11-12T09:29:39,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } 2023-11-12T09:29:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781379324 (current time:1699781379324). 2023-11-12T09:29:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:29:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781379323_ns3_test-16997811824172 VERSION not specified, setting to 2 2023-11-12T09:29:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa874ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:29:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:29:39,333 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:49622, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:29:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:29:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure2.ProcedureExecutor(1032): Stored pid=172, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } 2023-11-12T09:29:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 }, snapshot procedure id = 172 2023-11-12T09:29:39,337 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:29:39,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2023-11-12T09:29:39,337 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:29:39,339 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot, current working dir path: hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781379323_ns3_test-16997811824172 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:29:39,342 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:29:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2023-11-12T09:29:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2023-11-12T09:29:39,748 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:29:39,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure f50c319a7db3e25c53784414ac224095}] 2023-11-12T09:29:39,750 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure f50c319a7db3e25c53784414ac224095 2023-11-12T09:29:39,901 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:39,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36287 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=173 2023-11-12T09:29:39,902 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:29:39,902 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.HRegion(2520): Flush status journal for f50c319a7db3e25c53784414ac224095: 2023-11-12T09:29:39,902 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. for snapshot_1699781379323_ns3_test-16997811824172 completed. 2023-11-12T09:29:39,902 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095.' region-info for snapshot=snapshot_1699781379323_ns3_test-16997811824172 2023-11-12T09:29:39,902 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:29:39,902 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-12T09:29:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2023-11-12T09:29:40,307 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:29:40,307 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=173 2023-11-12T09:29:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.HMaster(4026): Remote procedure done, pid=173 2023-11-12T09:29:40,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781379323_ns3_test-16997811824172 on region f50c319a7db3e25c53784414ac224095 2023-11-12T09:29:40,309 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure f50c319a7db3e25c53784414ac224095 2023-11-12T09:29:40,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=173, resume processing ppid=172 2023-11-12T09:29:40,311 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:29:40,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=173, ppid=172, state=SUCCESS; SnapshotRegionProcedure f50c319a7db3e25c53784414ac224095 in 562 msec 2023-11-12T09:29:40,312 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:29:40,312 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:29:40,312 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781379323_ns3_test-16997811824172 2023-11-12T09:29:40,313 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781379323_ns3_test-16997811824172 2023-11-12T09:29:40,347 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-11-12T09:29:40,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2023-11-12T09:29:40,722 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:29:40,727 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:29:40,727 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/.tmp/snapshot_1699781379323_ns3_test-16997811824172 to hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781379323_ns3_test-16997811824172 2023-11-12T09:29:40,728 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:29:40,729 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 }, snapshot procedure id = 172 2023-11-12T09:29:40,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=172, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1699781379323_ns3_test-16997811824172 table=ns3:test-16997811824172 type=FLUSH ttl=0 } in 1.3950 sec 2023-11-12T09:29:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2023-11-12T09:29:41,442 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781372875 2023-11-12T09:29:41,442 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:29:41,442 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-12T09:29:41,442 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781372875_ns3:test-16997811824172 2023-11-12T09:29:41,442 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781379323_ns3_test-16997811824172 to hdfs://localhost:34955/backupUT/backup_1699781372875/ns3/test-16997811824172/ 2023-11-12T09:29:41,442 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:29:41,465 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:29:41,465 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_-1086299482_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781372875/ns3/test-16997811824172, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781372875/ns3/test-16997811824172/.hbase-snapshot/.tmp/snapshot_1699781379323_ns3_test-16997811824172 2023-11-12T09:29:41,465 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781379323_ns3_test-16997811824172. 2023-11-12T09:29:41,470 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781379323_ns3_test-16997811824172 to hdfs://localhost:34955/backupUT/backup_1699781372875/ns3/test-16997811824172/.hbase-snapshot/.tmp/snapshot_1699781379323_ns3_test-16997811824172 2023-11-12T09:29:41,885 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:41,885 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:41,886 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:41,886 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:42,868 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-5929517208737160783.jar 2023-11-12T09:29:42,868 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:42,868 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:42,869 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:42,869 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:42,870 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:42,870 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:42,870 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:42,870 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:42,871 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:42,871 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:42,872 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:29:42,872 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:29:42,873 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:29:42,873 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:29:42,874 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:29:42,874 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:29:42,874 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:29:42,875 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:29:42,875 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:29:42,875 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:29:42,876 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:29:42,876 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:42,877 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:42,877 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:42,878 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:42,878 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:42,878 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:42,879 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:42,925 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781379323_ns3_test-16997811824172' hfile list 2023-11-12T09:29:42,995 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781382961/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:29:42,995 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781382961/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:29:44,022 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:29:44,023 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:29:44,027 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781379323_ns3_test-16997811824172 2023-11-12T09:29:44,027 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781379323_ns3_test-16997811824172 finished. 2023-11-12T09:29:44,027 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781372875_ns2:test-16997811824171 2023-11-12T09:29:44,027 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781377203_ns2_test-16997811824171 to hdfs://localhost:34955/backupUT/backup_1699781372875/ns2/test-16997811824171/ 2023-11-12T09:29:44,027 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:29:44,050 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa 2023-11-12T09:29:44,050 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43639,1699781169211_995339036_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:34955/backupUT/backup_1699781372875/ns2/test-16997811824171, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:34955/backupUT/backup_1699781372875/ns2/test-16997811824171/.hbase-snapshot/.tmp/snapshot_1699781377203_ns2_test-16997811824171 2023-11-12T09:29:44,050 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:34955, inputRoot=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa, snapshotDir=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781377203_ns2_test-16997811824171. 2023-11-12T09:29:44,056 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/.hbase-snapshot/snapshot_1699781377203_ns2_test-16997811824171 to hdfs://localhost:34955/backupUT/backup_1699781372875/ns2/test-16997811824171/.hbase-snapshot/.tmp/snapshot_1699781377203_ns2_test-16997811824171 2023-11-12T09:29:44,474 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:44,475 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:44,476 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:44,476 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:45,448 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop-909753345056021645.jar 2023-11-12T09:29:45,448 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:45,449 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:45,450 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:45,450 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:45,450 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:45,451 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:45,451 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:45,451 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:45,451 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:29:45,452 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:29:45,453 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:29:45,453 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:29:45,453 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:29:45,454 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:29:45,454 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:29:45,455 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:29:45,455 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:29:45,455 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:29:45,456 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:29:45,456 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:29:45,456 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:29:45,457 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:45,457 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:45,458 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:45,458 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:45,458 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:29:45,459 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:45,460 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:29:45,502 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781377203_ns2_test-16997811824171' hfile list 2023-11-12T09:29:45,503 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-12T09:29:45,571 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781385538/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:29:45,572 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/mapreduce.cluster.local.dir/1699781385538/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:29:45,623 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:29:45,670 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_ output=hdfs://localhost:34955/backupUT/backup_1699781372875/ns2/test-16997811824171/archive/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/f/23111431b93e41f9be1d719ba383ef1a_SeqId_4_ 2023-11-12T09:29:45,670 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 4.160M/sec 2023-11-12T09:29:45,887 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:29:46,601 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:29:46,602 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:29:46,607 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781377203_ns2_test-16997811824171 2023-11-12T09:29:46,607 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781377203_ns2_test-16997811824171 finished. 2023-11-12T09:29:46,607 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): ns3:test-16997811824172 2023-11-12T09:29:46,608 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1034): ns2:test-16997811824171 2023-11-12T09:29:46,613 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781372875 2023-11-12T09:29:46,613 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:29:47,018 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781372875/.backup.manifest 2023-11-12T09:29:47,018 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781372875 2023-11-12T09:29:47,018 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:29:47,423 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:34955/backupUT/backup_1699781372875/.backup.manifest 2023-11-12T09:29:47,423 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(358): Backup backup_1699781372875 finished: type=FULL,tablelist=ns3:test-16997811824172;ns2:test-16997811824171,targetRootDir=hdfs://localhost:34955/backupUT,startts=1699781377018,completets=1699781386613,bytescopied=0 2023-11-12T09:29:47,423 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:29:47,423 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781379323_ns3_test-16997811824172 2023-11-12T09:29:47,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781379323_ns3_test-16997811824172" 2023-11-12T09:29:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781379323_ns3_test-16997811824172 2023-11-12T09:29:47,429 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781379323_ns3_test-16997811824172 for backup backup_1699781372875 succeeded. 2023-11-12T09:29:47,429 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781377203_ns2_test-16997811824171 2023-11-12T09:29:47,430 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781377203_ns2_test-16997811824171" 2023-11-12T09:29:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781377203_ns2_test-16997811824171 2023-11-12T09:29:47,432 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781377203_ns2_test-16997811824171 for backup backup_1699781372875 succeeded. 2023-11-12T09:29:47,433 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:29:47,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:29:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43639 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:29:47,438 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:29:47,439 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:29:47,444 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.TableBackupClient(376): Backup backup_1699781372875 completed. 2023-11-12T09:29:47,444 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:47,454 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-11-12T09:29:47,454 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-11-12T09:29:47,455 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781372875/.backup.manifest 2023-11-12T09:29:47,457 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781372875/.backup.manifest 2023-11-12T09:29:47,457 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781372875/.backup.manifest 2023-11-12T09:29:47,459 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781372875/.backup.manifest 2023-11-12T09:29:47,459 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:34955/backupUT/backup_1699781372875/.backup.manifest 2023-11-12T09:29:47,461 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781372875/.backup.manifest 2023-11-12T09:29:47,461 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] impl.RestoreTablesClient(148): Restoring 'faketable1' to 'ns2:table2' from full backup image hdfs://localhost:34955/backupUT/backup_1699781372875/default/faketable1 2023-11-12T09:29:47,479 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleDNE Thread=604 (was 605), OpenFileDescriptor=723 (was 713) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=112 (was 126), ProcessCount=169 (was 171), AvailableMemoryMB=5196 (was 5196) 2023-11-12T09:29:47,479 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-11-12T09:29:47,485 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-11-12T09:29:47,486 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:47,486 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-11-12T09:29:47,486 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] util.JVMClusterUtil(257): Found active master hash=1162968367, stopped=false 2023-11-12T09:29:47,486 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] master.ServerManager(896): Cluster shutdown requested of master=jenkins-hbase3.apache.org,43639,1699781169211 2023-11-12T09:29:47,492 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] procedure2.ProcedureExecutor(630): Stopping 2023-11-12T09:29:47,493 DEBUG [Listener at jenkins-hbase3.apache.org/36935 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:47,493 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,36287,1699781172353' ***** 2023-11-12T09:29:47,493 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-11-12T09:29:47,493 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-12T09:29:47,507 INFO [RS:0;jenkins-hbase3:36287 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@405ea105{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-12T09:29:47,512 INFO [RS:0;jenkins-hbase3:36287 {}] server.AbstractConnector(383): Stopped ServerConnector@342cc1cb{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:29:47,512 INFO [RS:0;jenkins-hbase3:36287 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-12T09:29:47,513 INFO [RS:0;jenkins-hbase3:36287 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@571c5e22{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-12T09:29:47,513 INFO [RS:0;jenkins-hbase3:36287 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@67098989{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop.log.dir/,STOPPED} 2023-11-12T09:29:47,516 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HeapMemoryManager(220): Stopping 2023-11-12T09:29:47,516 INFO [RS:0;jenkins-hbase3:36287 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-11-12T09:29:47,516 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-11-12T09:29:47,516 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-11-12T09:29:47,516 INFO [RS:0;jenkins-hbase3:36287 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-11-12T09:29:47,516 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(3528): Received CLOSE for f50c319a7db3e25c53784414ac224095 2023-11-12T09:29:47,516 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(3528): Received CLOSE for 606338a328e82acae51355d485ed81f0 2023-11-12T09:29:47,516 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(3528): Received CLOSE for 3ca9201e34d84739ee34b92f9445e4a2 2023-11-12T09:29:47,516 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(3528): Received CLOSE for 79ca523521e828a10ce7dfa8176221f4 2023-11-12T09:29:47,516 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(3528): Received CLOSE for d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:29:47,516 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(3528): Received CLOSE for 9276b632fc4549b20f67f1151f8dc56f 2023-11-12T09:29:47,516 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(3528): Received CLOSE for d347587386821efa96e22f590d908cde 2023-11-12T09:29:47,517 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,36287,1699781172353 2023-11-12T09:29:47,517 DEBUG [RS:0;jenkins-hbase3:36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:47,517 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-11-12T09:29:47,517 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-11-12T09:29:47,517 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-11-12T09:29:47,517 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-11-12T09:29:47,517 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing f50c319a7db3e25c53784414ac224095, disabling compactions & flushes 2023-11-12T09:29:47,517 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:29:47,517 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:29:47,517 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. after waiting 0 ms 2023-11-12T09:29:47,517 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:29:47,517 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-11-12T09:29:47,517 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1583): Online Regions={f50c319a7db3e25c53784414ac224095=ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095., 606338a328e82acae51355d485ed81f0=backup:system,,1699781191829.606338a328e82acae51355d485ed81f0., 1588230740=hbase:meta,,1.1588230740, 3ca9201e34d84739ee34b92f9445e4a2=test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2., 79ca523521e828a10ce7dfa8176221f4=hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4., d4d2b2f12334f3e59195d63ae19e1266=ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266., 9276b632fc4549b20f67f1151f8dc56f=backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f., d347587386821efa96e22f590d908cde=ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde.} 2023-11-12T09:29:47,518 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-12T09:29:47,518 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-12T09:29:47,518 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-12T09:29:47,518 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-12T09:29:47,518 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-12T09:29:47,521 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=38.00 KB heapSize=66.30 KB 2023-11-12T09:29:47,521 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3ca9201e34d84739ee34b92f9445e4a2, 606338a328e82acae51355d485ed81f0, 79ca523521e828a10ce7dfa8176221f4, 9276b632fc4549b20f67f1151f8dc56f, d347587386821efa96e22f590d908cde, d4d2b2f12334f3e59195d63ae19e1266, f50c319a7db3e25c53784414ac224095 2023-11-12T09:29:47,525 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns3/test-16997811824172/f50c319a7db3e25c53784414ac224095/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-12T09:29:47,526 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:47,527 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:29:47,527 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for f50c319a7db3e25c53784414ac224095: 2023-11-12T09:29:47,527 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16997811824172,,1699781325134.f50c319a7db3e25c53784414ac224095. 2023-11-12T09:29:47,527 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 606338a328e82acae51355d485ed81f0, disabling compactions & flushes 2023-11-12T09:29:47,527 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:29:47,527 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:29:47,527 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. after waiting 0 ms 2023-11-12T09:29:47,527 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:29:47,527 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 606338a328e82acae51355d485ed81f0 2/2 column families, dataSize=1.46 KB heapSize=2.61 KB 2023-11-12T09:29:47,530 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/7bbca8411e364239b1b25c5d57966c8d is 171, key is trslm:hdfs://localhost:34955/backupUT\x00ns2:test-16997811824171/meta:log-roll-map/1699781386610/Put/seqid=0 2023-11-12T09:29:47,550 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/.tmp/info/bf88fd8fa32b4cdcb83e99502acac75f is 159, key is ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde./info:regioninfo/1699781321163/Put/seqid=0 2023-11-12T09:29:47,551 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-11-12T09:29:47,555 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.30 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/.tmp/info/bf88fd8fa32b4cdcb83e99502acac75f 2023-11-12T09:29:47,572 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/.tmp/rep_barrier/c80bc33e9f384f44a30eb5f6e36b3660 is 107, key is ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b./rep_barrier:/1699781206107/DeleteFamily/seqid=0 2023-11-12T09:29:47,578 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/.tmp/rep_barrier/c80bc33e9f384f44a30eb5f6e36b3660 2023-11-12T09:29:47,596 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/.tmp/table/ab8055a17c1746cca49361c16e077132 is 101, key is ns2:test-16997811824171,,1699781185165.e1e466fdb640b701a3363f445eefa78b./table:/1699781206107/DeleteFamily/seqid=0 2023-11-12T09:29:47,721 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3ca9201e34d84739ee34b92f9445e4a2, 606338a328e82acae51355d485ed81f0, 79ca523521e828a10ce7dfa8176221f4, 9276b632fc4549b20f67f1151f8dc56f, d347587386821efa96e22f590d908cde, d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:29:47,922 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 3ca9201e34d84739ee34b92f9445e4a2, 606338a328e82acae51355d485ed81f0, 79ca523521e828a10ce7dfa8176221f4, 9276b632fc4549b20f67f1151f8dc56f, d347587386821efa96e22f590d908cde, d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:29:47,935 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/7bbca8411e364239b1b25c5d57966c8d 2023-11-12T09:29:47,946 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/ccfe618a9d514c91a91e1f343cd0c8ab is 524, key is session:backup_1699781372875/session:context/1699781387438/Put/seqid=0 2023-11-12T09:29:48,000 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.56 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/.tmp/table/ab8055a17c1746cca49361c16e077132 2023-11-12T09:29:48,006 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/.tmp/info/bf88fd8fa32b4cdcb83e99502acac75f as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/info/bf88fd8fa32b4cdcb83e99502acac75f 2023-11-12T09:29:48,011 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/info/bf88fd8fa32b4cdcb83e99502acac75f, entries=94, sequenceid=183, filesize=16.4 K 2023-11-12T09:29:48,012 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/.tmp/rep_barrier/c80bc33e9f384f44a30eb5f6e36b3660 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/rep_barrier/c80bc33e9f384f44a30eb5f6e36b3660 2023-11-12T09:29:48,016 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/rep_barrier/c80bc33e9f384f44a30eb5f6e36b3660, entries=12, sequenceid=183, filesize=6.5 K 2023-11-12T09:29:48,017 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/.tmp/table/ab8055a17c1746cca49361c16e077132 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/table/ab8055a17c1746cca49361c16e077132 2023-11-12T09:29:48,021 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/table/ab8055a17c1746cca49361c16e077132, entries=32, sequenceid=183, filesize=7.3 K 2023-11-12T09:29:48,022 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~38.00 KB/38910, heapSize ~66.25 KB/67840, currentSize=0 B/0 for 1588230740 in 504ms, sequenceid=183, compaction requested=false 2023-11-12T09:29:48,033 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/meta/1588230740/recovered.edits/186.seqid, newMaxSeqId=186, maxSeqId=1 2023-11-12T09:29:48,033 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:48,033 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-12T09:29:48,034 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-12T09:29:48,034 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-12T09:29:48,034 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-11-12T09:29:48,122 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1609): Waiting on 3ca9201e34d84739ee34b92f9445e4a2, 606338a328e82acae51355d485ed81f0, 79ca523521e828a10ce7dfa8176221f4, 9276b632fc4549b20f67f1151f8dc56f, d347587386821efa96e22f590d908cde, d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:29:48,156 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-11-12T09:29:48,156 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-11-12T09:29:48,322 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1609): Waiting on 3ca9201e34d84739ee34b92f9445e4a2, 606338a328e82acae51355d485ed81f0, 79ca523521e828a10ce7dfa8176221f4, 9276b632fc4549b20f67f1151f8dc56f, d347587386821efa96e22f590d908cde, d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:29:48,351 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=775 B at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/ccfe618a9d514c91a91e1f343cd0c8ab 2023-11-12T09:29:48,356 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/meta/7bbca8411e364239b1b25c5d57966c8d as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/7bbca8411e364239b1b25c5d57966c8d 2023-11-12T09:29:48,361 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/meta/7bbca8411e364239b1b25c5d57966c8d, entries=6, sequenceid=148, filesize=5.8 K 2023-11-12T09:29:48,361 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/.tmp/session/ccfe618a9d514c91a91e1f343cd0c8ab as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/ccfe618a9d514c91a91e1f343cd0c8ab 2023-11-12T09:29:48,366 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/session/ccfe618a9d514c91a91e1f343cd0c8ab, entries=2, sequenceid=148, filesize=5.5 K 2023-11-12T09:29:48,366 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~1.46 KB/1492, heapSize ~2.58 KB/2640, currentSize=0 B/0 for 606338a328e82acae51355d485ed81f0 in 839ms, sequenceid=148, compaction requested=true 2023-11-12T09:29:48,391 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system/606338a328e82acae51355d485ed81f0/recovered.edits/151.seqid, newMaxSeqId=151, maxSeqId=1 2023-11-12T09:29:48,391 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:48,392 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:29:48,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 606338a328e82acae51355d485ed81f0: 2023-11-12T09:29:48,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1699781191829.606338a328e82acae51355d485ed81f0. 2023-11-12T09:29:48,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 3ca9201e34d84739ee34b92f9445e4a2, disabling compactions & flushes 2023-11-12T09:29:48,392 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:29:48,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:29:48,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. after waiting 0 ms 2023-11-12T09:29:48,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:29:48,396 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/default/test-1699781182417/3ca9201e34d84739ee34b92f9445e4a2/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2023-11-12T09:29:48,396 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:48,397 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:29:48,397 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 3ca9201e34d84739ee34b92f9445e4a2: 2023-11-12T09:29:48,397 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1699781182417,,1699781267443.3ca9201e34d84739ee34b92f9445e4a2. 2023-11-12T09:29:48,397 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 79ca523521e828a10ce7dfa8176221f4, disabling compactions & flushes 2023-11-12T09:29:48,397 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. 2023-11-12T09:29:48,397 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. 2023-11-12T09:29:48,397 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. after waiting 0 ms 2023-11-12T09:29:48,397 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. 2023-11-12T09:29:48,397 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 79ca523521e828a10ce7dfa8176221f4 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-11-12T09:29:48,410 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/namespace/79ca523521e828a10ce7dfa8176221f4/.tmp/info/2f3a9eab67f24e01b9a34c09dbd98029 is 45, key is default/info:d/1699781177174/Put/seqid=0 2023-11-12T09:29:48,522 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1579): Waiting on 4 regions to close 2023-11-12T09:29:48,522 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1583): Online Regions={79ca523521e828a10ce7dfa8176221f4=hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4., d4d2b2f12334f3e59195d63ae19e1266=ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266., 9276b632fc4549b20f67f1151f8dc56f=backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f., d347587386821efa96e22f590d908cde=ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde.} 2023-11-12T09:29:48,522 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1609): Waiting on 79ca523521e828a10ce7dfa8176221f4, 9276b632fc4549b20f67f1151f8dc56f, d347587386821efa96e22f590d908cde, d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:29:48,723 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1609): Waiting on 79ca523521e828a10ce7dfa8176221f4, 9276b632fc4549b20f67f1151f8dc56f, d347587386821efa96e22f590d908cde, d4d2b2f12334f3e59195d63ae19e1266 2023-11-12T09:29:48,814 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/namespace/79ca523521e828a10ce7dfa8176221f4/.tmp/info/2f3a9eab67f24e01b9a34c09dbd98029 2023-11-12T09:29:48,822 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/namespace/79ca523521e828a10ce7dfa8176221f4/.tmp/info/2f3a9eab67f24e01b9a34c09dbd98029 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/namespace/79ca523521e828a10ce7dfa8176221f4/info/2f3a9eab67f24e01b9a34c09dbd98029 2023-11-12T09:29:48,827 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/namespace/79ca523521e828a10ce7dfa8176221f4/info/2f3a9eab67f24e01b9a34c09dbd98029, entries=7, sequenceid=11, filesize=5.1 K 2023-11-12T09:29:48,828 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 79ca523521e828a10ce7dfa8176221f4 in 431ms, sequenceid=11, compaction requested=false 2023-11-12T09:29:48,834 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/hbase/namespace/79ca523521e828a10ce7dfa8176221f4/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-11-12T09:29:48,834 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:48,835 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. 2023-11-12T09:29:48,835 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 79ca523521e828a10ce7dfa8176221f4: 2023-11-12T09:29:48,835 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1699781176592.79ca523521e828a10ce7dfa8176221f4. 2023-11-12T09:29:48,836 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing d4d2b2f12334f3e59195d63ae19e1266, disabling compactions & flushes 2023-11-12T09:29:48,836 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. 2023-11-12T09:29:48,836 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. 2023-11-12T09:29:48,836 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. after waiting 0 ms 2023-11-12T09:29:48,836 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. 2023-11-12T09:29:48,839 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns4/test-16997811824173/d4d2b2f12334f3e59195d63ae19e1266/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:29:48,839 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:48,840 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. 2023-11-12T09:29:48,840 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for d4d2b2f12334f3e59195d63ae19e1266: 2023-11-12T09:29:48,840 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16997811824173,,1699781189500.d4d2b2f12334f3e59195d63ae19e1266. 2023-11-12T09:29:48,842 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9276b632fc4549b20f67f1151f8dc56f, disabling compactions & flushes 2023-11-12T09:29:48,842 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. 2023-11-12T09:29:48,842 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. 2023-11-12T09:29:48,842 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. after waiting 0 ms 2023-11-12T09:29:48,842 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. 2023-11-12T09:29:48,842 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 9276b632fc4549b20f67f1151f8dc56f 2/2 column families, dataSize=5.66 KB heapSize=8.50 KB 2023-11-12T09:29:48,857 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system_bulk/9276b632fc4549b20f67f1151f8dc56f/.tmp/meta/4b460a23c1dc45ae8f252f15d26116e7 is 323, key is bulk:ns2:test-16997811824171:6e68e35ec82e3efaab2edd4a90156b95:41d6c8458a9d4384a0a6faa0e53d84b4_SeqId_4_/meta:path/1699781211167/Put/seqid=0 2023-11-12T09:29:48,923 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1609): Waiting on 9276b632fc4549b20f67f1151f8dc56f, d347587386821efa96e22f590d908cde 2023-11-12T09:29:49,123 DEBUG [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1609): Waiting on 9276b632fc4549b20f67f1151f8dc56f, d347587386821efa96e22f590d908cde 2023-11-12T09:29:49,264 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.66 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system_bulk/9276b632fc4549b20f67f1151f8dc56f/.tmp/meta/4b460a23c1dc45ae8f252f15d26116e7 2023-11-12T09:29:49,270 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system_bulk/9276b632fc4549b20f67f1151f8dc56f/.tmp/meta/4b460a23c1dc45ae8f252f15d26116e7 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system_bulk/9276b632fc4549b20f67f1151f8dc56f/meta/4b460a23c1dc45ae8f252f15d26116e7 2023-11-12T09:29:49,274 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system_bulk/9276b632fc4549b20f67f1151f8dc56f/meta/4b460a23c1dc45ae8f252f15d26116e7, entries=16, sequenceid=12, filesize=8.1 K 2023-11-12T09:29:49,275 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~5.66 KB/5800, heapSize ~8.23 KB/8432, currentSize=0 B/0 for 9276b632fc4549b20f67f1151f8dc56f in 433ms, sequenceid=12, compaction requested=false 2023-11-12T09:29:49,281 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/backup/system_bulk/9276b632fc4549b20f67f1151f8dc56f/recovered.edits/15.seqid, newMaxSeqId=15, maxSeqId=1 2023-11-12T09:29:49,282 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:49,282 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. 2023-11-12T09:29:49,282 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9276b632fc4549b20f67f1151f8dc56f: 2023-11-12T09:29:49,282 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1699781192955.9276b632fc4549b20f67f1151f8dc56f. 2023-11-12T09:29:49,282 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing d347587386821efa96e22f590d908cde, disabling compactions & flushes 2023-11-12T09:29:49,282 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:29:49,282 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:29:49,282 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. after waiting 0 ms 2023-11-12T09:29:49,282 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:29:49,285 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/data/ns2/test-16997811824171/d347587386821efa96e22f590d908cde/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2023-11-12T09:29:49,286 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:49,286 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:29:49,286 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for d347587386821efa96e22f590d908cde: 2023-11-12T09:29:49,286 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16997811824171,,1699781320008.d347587386821efa96e22f590d908cde. 2023-11-12T09:29:49,323 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,36287,1699781172353; all regions closed. 2023-11-12T09:29:49,335 DEBUG [RS:0;jenkins-hbase3:36287 {}] wal.AbstractFSWAL(1062): Moved 10 WAL file(s) to /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs 2023-11-12T09:29:49,335 INFO [RS:0;jenkins-hbase3:36287 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353.meta:.meta(num 1699781377053) 2023-11-12T09:29:49,344 DEBUG [RS:0;jenkins-hbase3:36287 {}] wal.AbstractFSWAL(1062): Moved 7 WAL file(s) to /user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/oldWALs 2023-11-12T09:29:49,344 INFO [RS:0;jenkins-hbase3:36287 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C36287%2C1699781172353:(num 1699781377036) 2023-11-12T09:29:49,344 DEBUG [RS:0;jenkins-hbase3:36287 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:49,344 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.LeaseManager(133): Closed leases 2023-11-12T09:29:49,345 INFO [RS:0;jenkins-hbase3:36287 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-11-12T09:29:49,345 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-12T09:29:49,346 INFO [RS:0;jenkins-hbase3:36287 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:36287 2023-11-12T09:29:49,356 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,36287,1699781172353] 2023-11-12T09:29:49,357 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,36287,1699781172353; numProcessing=1 2023-11-12T09:29:49,359 INFO [RegionServerTracker-0 {}] master.ServerManager(561): Cluster shutdown set; jenkins-hbase3.apache.org,36287,1699781172353 expired; onlineServers=0 2023-11-12T09:29:49,359 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,43639,1699781169211' ***** 2023-11-12T09:29:49,360 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-11-12T09:29:49,360 DEBUG [M:0;jenkins-hbase3:43639 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17d0b820, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-11-12T09:29:49,360 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-12T09:29:49,408 INFO [M:0;jenkins-hbase3:43639 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@65329596{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-12T09:29:49,408 INFO [M:0;jenkins-hbase3:43639 {}] server.AbstractConnector(383): Stopped ServerConnector@45e81754{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:29:49,408 INFO [M:0;jenkins-hbase3:43639 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-12T09:29:49,409 INFO [M:0;jenkins-hbase3:43639 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3645ed1f{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-12T09:29:49,410 INFO [M:0;jenkins-hbase3:43639 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@56e2e346{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/hadoop.log.dir/,STOPPED} 2023-11-12T09:29:49,410 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,43639,1699781169211 2023-11-12T09:29:49,410 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,43639,1699781169211; all regions closed. 2023-11-12T09:29:49,410 DEBUG [M:0;jenkins-hbase3:43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:49,410 INFO [M:0;jenkins-hbase3:43639 {}] master.HMaster(1666): Stopping master jetty server 2023-11-12T09:29:49,411 INFO [M:0;jenkins-hbase3:43639 {}] server.AbstractConnector(383): Stopped ServerConnector@28909016{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:29:49,411 DEBUG [M:0;jenkins-hbase3:43639 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-11-12T09:29:49,411 INFO [M:0;jenkins-hbase3:43639 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-11-12T09:29:49,411 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-11-12T09:29:49,411 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781174965 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781174965,5,FailOnTimeoutGroup] 2023-11-12T09:29:49,411 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781174966 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781174966,5,FailOnTimeoutGroup] 2023-11-12T09:29:49,411 DEBUG [M:0;jenkins-hbase3:43639 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:29:49,412 DEBUG [M:0;jenkins-hbase3:43639 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-11-12T09:29:49,412 INFO [M:0;jenkins-hbase3:43639 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-11-12T09:29:49,412 DEBUG [M:0;jenkins-hbase3:43639 {}] master.HMaster(1689): Stopping service threads 2023-11-12T09:29:49,412 INFO [M:0;jenkins-hbase3:43639 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-11-12T09:29:49,412 ERROR [M:0;jenkins-hbase3:43639 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-12,5,PEWorkerGroup] 2023-11-12T09:29:49,413 INFO [M:0;jenkins-hbase3:43639 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-11-12T09:29:49,413 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-11-12T09:29:49,413 WARN [M:0;jenkins-hbase3:43639 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-11-12T09:29:49,413 INFO [M:0;jenkins-hbase3:43639 {}] assignment.AssignmentManager(383): Stopping assignment manager 2023-11-12T09:29:49,413 INFO [M:0;jenkins-hbase3:43639 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-11-12T09:29:49,414 DEBUG [M:0;jenkins-hbase3:43639 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-12T09:29:49,425 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:29:49,425 DEBUG [M:0;jenkins-hbase3:43639 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:29:49,425 DEBUG [M:0;jenkins-hbase3:43639 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-12T09:29:49,425 DEBUG [M:0;jenkins-hbase3:43639 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:29:49,425 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=621.89 KB heapSize=756.40 KB 2023-11-12T09:29:49,439 DEBUG [M:0;jenkins-hbase3:43639 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/893687be0095493b94b8bfc3d527d8c7 is 88, key is hbase:meta,,1/info:sn/1699781175965/Put/seqid=0 2023-11-12T09:29:49,456 INFO [RS:0;jenkins-hbase3:36287 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,36287,1699781172353; zookeeper connection closed. 2023-11-12T09:29:49,456 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2110a3a5 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2110a3a5 2023-11-12T09:29:49,456 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-11-12T09:29:49,844 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=1616 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/893687be0095493b94b8bfc3d527d8c7 2023-11-12T09:29:49,862 DEBUG [M:0;jenkins-hbase3:43639 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e36ae63bedd4177bd7449901441e2dc is 987, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1699781194158/Put/seqid=0 2023-11-12T09:29:50,266 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=621.29 KB at sequenceid=1616 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e36ae63bedd4177bd7449901441e2dc 2023-11-12T09:29:50,271 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e36ae63bedd4177bd7449901441e2dc 2023-11-12T09:29:50,284 DEBUG [M:0;jenkins-hbase3:43639 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/39e87b44bec14498b14c29a11489baef is 82, key is jenkins-hbase3.apache.org,36287,1699781172353/rs:state/1699781175052/Put/seqid=0 2023-11-12T09:29:50,347 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997811824171 2023-11-12T09:29:50,347 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-12T09:29:50,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-12T09:29:50,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-11-12T09:29:50,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-11-12T09:29:50,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-11-12T09:29:50,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16997811824173 2023-11-12T09:29:50,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16997811824172 2023-11-12T09:29:50,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:29:50,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1699781182417 2023-11-12T09:29:50,688 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=1616 (bloomFilter=true), to=hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/39e87b44bec14498b14c29a11489baef 2023-11-12T09:29:50,694 DEBUG [M:0;jenkins-hbase3:43639 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/893687be0095493b94b8bfc3d527d8c7 as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/893687be0095493b94b8bfc3d527d8c7 2023-11-12T09:29:50,698 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/893687be0095493b94b8bfc3d527d8c7, entries=8, sequenceid=1616, filesize=5.5 K 2023-11-12T09:29:50,699 DEBUG [M:0;jenkins-hbase3:43639 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1e36ae63bedd4177bd7449901441e2dc as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e36ae63bedd4177bd7449901441e2dc 2023-11-12T09:29:50,703 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1e36ae63bedd4177bd7449901441e2dc 2023-11-12T09:29:50,703 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1e36ae63bedd4177bd7449901441e2dc, entries=173, sequenceid=1616, filesize=39.1 K 2023-11-12T09:29:50,704 DEBUG [M:0;jenkins-hbase3:43639 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/39e87b44bec14498b14c29a11489baef as hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/39e87b44bec14498b14c29a11489baef 2023-11-12T09:29:50,707 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:34955/user/jenkins/test-data/7ff5789a-5342-659a-1313-0a3bf9b8e7fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/39e87b44bec14498b14c29a11489baef, entries=1, sequenceid=1616, filesize=5.1 K 2023-11-12T09:29:50,708 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.HRegion(3022): Finished flush of dataSize ~621.89 KB/636811, heapSize ~756.10 KB/774248, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1283ms, sequenceid=1616, compaction requested=false 2023-11-12T09:29:50,713 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:29:50,713 DEBUG [M:0;jenkins-hbase3:43639 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:29:50,716 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-12T09:29:50,716 INFO [M:0;jenkins-hbase3:43639 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-11-12T09:29:50,716 INFO [M:0;jenkins-hbase3:43639 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-11-12T09:29:50,717 INFO [M:0;jenkins-hbase3:43639 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:43639 2023-11-12T09:29:50,820 INFO [M:0;jenkins-hbase3:43639 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,43639,1699781169211; zookeeper connection closed. 2023-11-12T09:29:50,828 WARN [Listener at jenkins-hbase3.apache.org/36935 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-12T09:29:50,834 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:29:50,942 WARN [BP-1279779094-172.31.12.81-1699781165767 heartbeating to localhost/127.0.0.1:34955 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-12T09:29:50,942 WARN [BP-1279779094-172.31.12.81-1699781165767 heartbeating to localhost/127.0.0.1:34955 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1279779094-172.31.12.81-1699781165767 (Datanode Uuid d9010a15-a2e1-4fc3-a1cc-a04c2149c14d) service to localhost/127.0.0.1:34955 2023-11-12T09:29:50,944 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/cluster_2df797c1-14cb-183c-6424-e9a9271897e8/dfs/data/data1/current/BP-1279779094-172.31.12.81-1699781165767 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:29:50,945 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a37e2741-413f-8124-ef95-e49c9dfb58fb/cluster_2df797c1-14cb-183c-6424-e9a9271897e8/dfs/data/data2/current/BP-1279779094-172.31.12.81-1699781165767 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:29:50,976 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:29:51,207 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-11-12T09:29:51,207 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-11-12T09:29:51,218 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:29:51,332 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:29:51,441 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:29:51,442 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:29:51,547 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-11-12T09:29:51,553 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-11-12T09:29:51,555 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:29:51,558 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:29:51,661 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:29:51,661 INFO [Listener at jenkins-hbase3.apache.org/36935 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped