2023-10-02 12:34:16,263 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 2023-10-02 12:34:16,283 main DEBUG Took 0.016304 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-10-02 12:34:16,283 main DEBUG PluginManager 'Core' found 129 plugins 2023-10-02 12:34:16,283 main DEBUG PluginManager 'Level' found 0 plugins 2023-10-02 12:34:16,284 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-10-02 12:34:16,286 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:34:16,296 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-10-02 12:34:16,313 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,316 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:34:16,316 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,317 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:34:16,317 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,318 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:34:16,319 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,319 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:34:16,320 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,320 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:34:16,321 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,321 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:34:16,322 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,322 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:34:16,323 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,323 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:34:16,324 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,324 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:34:16,324 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,325 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:34:16,325 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,326 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:34:16,326 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,327 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:34:16,327 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,327 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-10-02 12:34:16,330 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:34:16,333 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-10-02 12:34:16,335 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-10-02 12:34:16,336 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-10-02 12:34:16,337 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-10-02 12:34:16,337 main DEBUG PluginManager 'Converter' found 47 plugins 2023-10-02 12:34:16,354 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-10-02 12:34:16,358 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-10-02 12:34:16,363 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-10-02 12:34:16,363 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-10-02 12:34:16,364 main DEBUG createAppenders(={Console}) 2023-10-02 12:34:16,365 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 initialized 2023-10-02 12:34:16,365 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 2023-10-02 12:34:16,366 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@13c9d689 OK. 2023-10-02 12:34:16,366 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-10-02 12:34:16,367 main DEBUG OutputStream closed 2023-10-02 12:34:16,367 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-10-02 12:34:16,367 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-10-02 12:34:16,368 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@15c43bd9 OK 2023-10-02 12:34:16,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-10-02 12:34:16,426 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-10-02 12:34:16,427 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-10-02 12:34:16,428 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-10-02 12:34:16,429 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-10-02 12:34:16,429 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-10-02 12:34:16,430 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-10-02 12:34:16,430 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-10-02 12:34:16,431 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-10-02 12:34:16,431 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-10-02 12:34:16,432 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-10-02 12:34:16,432 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-10-02 12:34:16,432 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-10-02 12:34:16,433 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-10-02 12:34:16,433 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-10-02 12:34:16,434 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-10-02 12:34:16,434 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-10-02 12:34:16,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-10-02 12:34:16,439 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02 12:34:16,439 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@1b6e1eff) with optional ClassLoader: null 2023-10-02 12:34:16,439 main DEBUG Shutdown hook enabled. Registering a new one. 2023-10-02 12:34:16,441 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@1b6e1eff] started OK. 2023-10-02T12:34:16,471 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupMultipleDeletes timeout: 13 mins 2023-10-02 12:34:16,476 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-10-02 12:34:16,476 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02T12:34:16,924 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07 2023-10-02T12:34:16,929 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:34:16,931 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:16,932 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-10-02T12:34:16,936 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/cluster_9dfd02cb-0bbd-743a-fb9e-a562fae510de, deleteOnExit=true 2023-10-02T12:34:16,936 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-10-02T12:34:16,936 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/test.cache.data in system properties and HBase conf 2023-10-02T12:34:16,937 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:34:16,938 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:34:16,938 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:34:16,938 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:34:16,939 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:34:17,058 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-10-02T12:34:17,639 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-02T12:34:17,644 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:34:17,645 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:34:17,645 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:34:17,645 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:34:17,646 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:34:17,646 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:34:17,646 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:34:17,647 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:34:17,647 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:34:17,647 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:34:17,647 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:34:17,648 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:34:17,648 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:34:17,648 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-10-02T12:34:18,191 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:34:18,195 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:34:18,520 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-10-02T12:34:18,682 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@1f30304b via org.mortbay.log.Slf4jLog 2023-10-02T12:34:18,698 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:34:18,740 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:34:18,796 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/java.io.tmpdir/Jetty_localhost_41179_hdfs____9behgg/webapp 2023-10-02T12:34:18,983 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41179 2023-10-02T12:34:19,020 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:34:19,020 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:34:19,593 WARN [Listener at localhost/33727 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:34:19,661 WARN [Listener at localhost/33727 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-02T12:34:19,681 WARN [Listener at localhost/33727 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:34:19,687 INFO [Listener at localhost/33727 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:34:19,694 INFO [Listener at localhost/33727 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/java.io.tmpdir/Jetty_localhost_43715_datanode____.cclc9p/webapp 2023-10-02T12:34:19,804 INFO [Listener at localhost/33727 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:43715 2023-10-02T12:34:20,199 WARN [Listener at localhost/34813 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:34:20,516 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-02T12:34:20,787 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xc9f68d19568ec38a: Processing first storage report for DS-6b753d83-936a-4b84-b000-e87f03b1a904 from datanode 0f2caf88-88c6-4ff7-a391-75aaacc42dcb 2023-10-02T12:34:20,788 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xc9f68d19568ec38a: from storage DS-6b753d83-936a-4b84-b000-e87f03b1a904 node DatanodeRegistration(127.0.0.1:36399, datanodeUuid=0f2caf88-88c6-4ff7-a391-75aaacc42dcb, infoPort=34633, infoSecurePort=0, ipcPort=34813, storageInfo=lv=-57;cid=testClusterID;nsid=1359560098;c=1696250058286), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-10-02T12:34:20,789 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xc9f68d19568ec38a: Processing first storage report for DS-ae8025ed-4f8b-49c8-8f94-69e7d30339c5 from datanode 0f2caf88-88c6-4ff7-a391-75aaacc42dcb 2023-10-02T12:34:20,789 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xc9f68d19568ec38a: from storage DS-ae8025ed-4f8b-49c8-8f94-69e7d30339c5 node DatanodeRegistration(127.0.0.1:36399, datanodeUuid=0f2caf88-88c6-4ff7-a391-75aaacc42dcb, infoPort=34633, infoSecurePort=0, ipcPort=34813, storageInfo=lv=-57;cid=testClusterID;nsid=1359560098;c=1696250058286), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:34:20,864 DEBUG [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07 2023-10-02T12:34:21,004 INFO [Listener at localhost/34813 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/cluster_9dfd02cb-0bbd-743a-fb9e-a562fae510de/zookeeper_0, clientPort=64600, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/cluster_9dfd02cb-0bbd-743a-fb9e-a562fae510de/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/cluster_9dfd02cb-0bbd-743a-fb9e-a562fae510de/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-10-02T12:34:21,030 INFO [Listener at localhost/34813 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=64600 2023-10-02T12:34:21,042 INFO [Listener at localhost/34813 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:34:21,045 INFO [Listener at localhost/34813 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:34:21,794 INFO [Listener at localhost/34813 {}] util.FSUtils(462): Created version file at hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3 with version=8 2023-10-02T12:34:21,794 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/hbase-staging 2023-10-02T12:34:21,812 DEBUG [Listener at localhost/34813 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-10-02T12:34:21,812 DEBUG [Listener at localhost/34813 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-10-02T12:34:21,813 DEBUG [Listener at localhost/34813 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-10-02T12:34:21,813 DEBUG [Listener at localhost/34813 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-10-02T12:34:22,037 DEBUG [Listener at localhost/34813 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-10-02T12:34:22,881 INFO [Listener at localhost/34813 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:34:22,932 INFO [Listener at localhost/34813 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:34:22,933 INFO [Listener at localhost/34813 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:34:22,933 INFO [Listener at localhost/34813 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:34:22,933 INFO [Listener at localhost/34813 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:34:22,933 INFO [Listener at localhost/34813 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:34:23,147 INFO [Listener at localhost/34813 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:34:23,369 INFO [Listener at localhost/34813 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-10-02T12:34:23,408 DEBUG [Listener at localhost/34813 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-10-02T12:34:23,417 INFO [Listener at localhost/34813 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:34:23,477 DEBUG [Listener at localhost/34813 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 3001 (auto-detected) 2023-10-02T12:34:23,479 DEBUG [Listener at localhost/34813 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-10-02T12:34:23,520 INFO [Listener at localhost/34813 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:39919 2023-10-02T12:34:23,536 INFO [Listener at localhost/34813 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:34:23,539 INFO [Listener at localhost/34813 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:34:23,566 INFO [Listener at localhost/34813 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:39919 connecting to ZooKeeper ensemble=127.0.0.1:64600 2023-10-02T12:34:23,640 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:399190x0, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:34:23,645 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:39919-0x101732b40f20000 connected 2023-10-02T12:34:23,707 DEBUG [Listener at localhost/34813 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:34:23,712 DEBUG [Listener at localhost/34813 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:34:23,716 DEBUG [Listener at localhost/34813 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:34:23,739 DEBUG [Listener at localhost/34813 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39919 2023-10-02T12:34:23,740 DEBUG [Listener at localhost/34813 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39919 2023-10-02T12:34:23,741 DEBUG [Listener at localhost/34813 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39919 2023-10-02T12:34:23,742 DEBUG [Listener at localhost/34813 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39919 2023-10-02T12:34:23,745 DEBUG [Listener at localhost/34813 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39919 2023-10-02T12:34:23,808 INFO [Listener at localhost/34813 {}] log.Log(170): Logging initialized @8610ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-10-02T12:34:23,959 INFO [Listener at localhost/34813 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:34:23,960 INFO [Listener at localhost/34813 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:34:23,961 INFO [Listener at localhost/34813 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:34:23,964 INFO [Listener at localhost/34813 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-10-02T12:34:23,965 INFO [Listener at localhost/34813 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:34:23,965 INFO [Listener at localhost/34813 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:34:23,969 INFO [Listener at localhost/34813 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:34:24,137 INFO [Listener at localhost/34813 {}] http.HttpServer(1219): Jetty bound to port 38657 2023-10-02T12:34:24,140 INFO [Listener at localhost/34813 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:34:24,204 INFO [Listener at localhost/34813 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:34:24,207 INFO [Listener at localhost/34813 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1c0a99ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:34:24,208 INFO [Listener at localhost/34813 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:34:24,208 INFO [Listener at localhost/34813 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@777c649c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:34:24,297 INFO [Listener at localhost/34813 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:34:24,310 INFO [Listener at localhost/34813 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:34:24,311 INFO [Listener at localhost/34813 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:34:24,313 INFO [Listener at localhost/34813 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-10-02T12:34:24,321 INFO [Listener at localhost/34813 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:34:25,692 INFO [Listener at localhost/34813 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@673fe3{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:34:25,704 INFO [Listener at localhost/34813 {}] server.AbstractConnector(333): Started ServerConnector@3ecfb4c5{HTTP/1.1, (http/1.1)}{0.0.0.0:38657} 2023-10-02T12:34:25,705 INFO [Listener at localhost/34813 {}] server.Server(415): Started @10507ms 2023-10-02T12:34:25,708 INFO [Listener at localhost/34813 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3, hbase.cluster.distributed=false 2023-10-02T12:34:25,771 INFO [Listener at localhost/34813 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:34:25,772 INFO [Listener at localhost/34813 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:34:25,772 INFO [Listener at localhost/34813 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:34:25,772 INFO [Listener at localhost/34813 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:34:25,772 INFO [Listener at localhost/34813 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:34:25,772 INFO [Listener at localhost/34813 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:34:25,781 INFO [Listener at localhost/34813 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:34:25,783 INFO [Listener at localhost/34813 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:34:25,784 INFO [Listener at localhost/34813 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:40211 2023-10-02T12:34:25,787 INFO [Listener at localhost/34813 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-10-02T12:34:25,795 DEBUG [Listener at localhost/34813 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-10-02T12:34:25,796 INFO [Listener at localhost/34813 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:34:25,799 INFO [Listener at localhost/34813 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:34:25,801 INFO [Listener at localhost/34813 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:40211 connecting to ZooKeeper ensemble=127.0.0.1:64600 2023-10-02T12:34:25,805 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:402110x0, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:34:25,806 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:40211-0x101732b40f20001 connected 2023-10-02T12:34:25,806 DEBUG [Listener at localhost/34813 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:34:25,808 DEBUG [Listener at localhost/34813 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:34:25,808 DEBUG [Listener at localhost/34813 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:34:25,809 DEBUG [Listener at localhost/34813 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40211 2023-10-02T12:34:25,810 DEBUG [Listener at localhost/34813 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40211 2023-10-02T12:34:25,810 DEBUG [Listener at localhost/34813 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40211 2023-10-02T12:34:25,811 DEBUG [Listener at localhost/34813 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40211 2023-10-02T12:34:25,811 DEBUG [Listener at localhost/34813 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40211 2023-10-02T12:34:25,816 INFO [Listener at localhost/34813 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:34:25,816 INFO [Listener at localhost/34813 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:34:25,816 INFO [Listener at localhost/34813 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:34:25,818 INFO [Listener at localhost/34813 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-10-02T12:34:25,818 INFO [Listener at localhost/34813 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:34:25,818 INFO [Listener at localhost/34813 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:34:25,818 INFO [Listener at localhost/34813 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:34:25,821 INFO [Listener at localhost/34813 {}] http.HttpServer(1219): Jetty bound to port 33689 2023-10-02T12:34:25,822 INFO [Listener at localhost/34813 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:34:25,823 INFO [Listener at localhost/34813 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:34:25,824 INFO [Listener at localhost/34813 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@4a19e3c9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:34:25,824 INFO [Listener at localhost/34813 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:34:25,825 INFO [Listener at localhost/34813 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@640bb327{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:34:25,843 INFO [Listener at localhost/34813 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:34:25,844 INFO [Listener at localhost/34813 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:34:25,845 INFO [Listener at localhost/34813 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:34:25,845 INFO [Listener at localhost/34813 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-10-02T12:34:25,847 INFO [Listener at localhost/34813 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:34:25,852 INFO [Listener at localhost/34813 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@245e1c35{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:34:25,853 INFO [Listener at localhost/34813 {}] server.AbstractConnector(333): Started ServerConnector@4b8b4081{HTTP/1.1, (http/1.1)}{0.0.0.0:33689} 2023-10-02T12:34:25,854 INFO [Listener at localhost/34813 {}] server.Server(415): Started @10656ms 2023-10-02T12:34:25,859 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:34:25,868 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@1fdf5b65{HTTP/1.1, (http/1.1)}{0.0.0.0:40343} 2023-10-02T12:34:25,868 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @10671ms 2023-10-02T12:34:25,868 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,39919,1696250062021 2023-10-02T12:34:25,883 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:34:25,883 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:34:25,887 DEBUG [M:0;jenkins-hbase3:39919 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:39919 2023-10-02T12:34:25,890 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,39919,1696250062021 2023-10-02T12:34:25,919 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:34:25,919 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:34:25,920 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:25,920 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:25,921 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:34:25,923 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,39919,1696250062021 from backup master directory 2023-10-02T12:34:25,923 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:34:25,933 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,39919,1696250062021 2023-10-02T12:34:25,933 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:34:25,933 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:34:25,934 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:34:25,934 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,39919,1696250062021 2023-10-02T12:34:25,938 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-10-02T12:34:25,940 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-10-02T12:34:26,030 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/hbase.id with ID: 2cc659f3-8788-490e-81ca-10c6d9410c4f 2023-10-02T12:34:26,073 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:34:26,090 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:26,090 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:26,144 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:34:26,147 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-10-02T12:34:26,172 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-10-02T12:34:26,172 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-10-02T12:34:26,174 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:34:26,185 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:34:26,187 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:34:26,678 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store 2023-10-02T12:34:26,738 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-10-02T12:34:26,739 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:26,739 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:34:26,739 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:26,739 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:26,740 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2023-10-02T12:34:26,740 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:26,740 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:34:26,740 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:34:26,745 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/.initializing 2023-10-02T12:34:26,745 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/WALs/jenkins-hbase3.apache.org,39919,1696250062021 2023-10-02T12:34:26,759 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:34:26,782 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C39919%2C1696250062021, suffix=, logDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/WALs/jenkins-hbase3.apache.org,39919,1696250062021, archiveDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/oldWALs, maxLogs=10 2023-10-02T12:34:26,829 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/WALs/jenkins-hbase3.apache.org,39919,1696250062021/jenkins-hbase3.apache.org%2C39919%2C1696250062021.1696250066792, exclude list is [], retry=0 2023-10-02T12:34:26,872 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:34:26,884 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:34:26,974 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/WALs/jenkins-hbase3.apache.org,39919,1696250062021/jenkins-hbase3.apache.org%2C39919%2C1696250062021.1696250066792 2023-10-02T12:34:26,975 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:34:26,976 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:34:26,976 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:26,981 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:34:26,983 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:34:27,057 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:34:27,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-10-02T12:34:27,151 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:27,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:34:27,156 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:34:27,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-10-02T12:34:27,159 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:27,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:34:27,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:34:27,164 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-10-02T12:34:27,164 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:27,165 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:34:27,165 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:34:27,168 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-10-02T12:34:27,168 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:27,169 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:34:27,173 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:34:27,174 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:34:27,189 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-10-02T12:34:27,193 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:34:27,197 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:34:27,198 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10235381920, jitterRate=-0.046755775809288025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-10-02T12:34:27,204 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:34:27,206 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-10-02T12:34:27,243 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69792f74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:27,304 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-10-02T12:34:27,336 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-10-02T12:34:27,336 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-10-02T12:34:27,341 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-10-02T12:34:27,343 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-10-02T12:34:27,352 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 8 msec 2023-10-02T12:34:27,352 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-10-02T12:34:27,397 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-10-02T12:34:27,415 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-10-02T12:34:27,419 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-10-02T12:34:27,423 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-10-02T12:34:27,424 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-10-02T12:34:27,425 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-10-02T12:34:27,429 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-10-02T12:34:27,435 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-10-02T12:34:27,437 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-10-02T12:34:27,438 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-10-02T12:34:27,439 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-10-02T12:34:27,455 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-10-02T12:34:27,458 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-10-02T12:34:27,463 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:34:27,463 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:34:27,464 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:27,464 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:27,464 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,39919,1696250062021, sessionid=0x101732b40f20000, setting cluster-up flag (Was=false) 2023-10-02T12:34:27,474 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-10-02T12:34:27,482 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:27,482 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:27,488 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-10-02T12:34:27,490 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,39919,1696250062021 2023-10-02T12:34:27,495 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:27,495 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:27,500 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-10-02T12:34:27,501 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,39919,1696250062021 2023-10-02T12:34:27,507 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:27,507 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:27,512 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-10-02T12:34:27,513 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,39919,1696250062021 2023-10-02T12:34:27,576 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:40211 2023-10-02T12:34:27,577 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(997): ClusterId : 2cc659f3-8788-490e-81ca-10c6d9410c4f 2023-10-02T12:34:27,580 INFO [RS:0;jenkins-hbase3:40211 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-10-02T12:34:27,584 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-10-02T12:34:27,591 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-10-02T12:34:27,591 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-10-02T12:34:27,595 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-10-02T12:34:27,595 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-10-02T12:34:27,599 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-10-02T12:34:27,600 DEBUG [RS:0;jenkins-hbase3:40211 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fbcff71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:27,602 DEBUG [RS:0;jenkins-hbase3:40211 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289db9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:34:27,607 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-10-02T12:34:27,607 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-10-02T12:34:27,607 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-10-02T12:34:27,610 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,39919,1696250062021 with isa=jenkins-hbase3.apache.org/172.31.12.81:40211, startcode=1696250065770 2023-10-02T12:34:27,634 DEBUG [RS:0;jenkins-hbase3:40211 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:34:27,701 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-10-02T12:34:27,704 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55171, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:34:27,712 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-10-02T12:34:27,715 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39919 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3181) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:590) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-02T12:34:27,717 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-10-02T12:34:27,729 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,39919,1696250062021 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-10-02T12:34:27,735 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:34:27,735 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:34:27,735 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:34:27,735 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:34:27,735 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-10-02T12:34:27,735 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:34:27,736 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:34:27,736 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:34:27,760 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-10-02T12:34:27,761 WARN [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-10-02T12:34:27,769 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1696250097768 2023-10-02T12:34:27,774 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-10-02T12:34:27,775 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:34:27,776 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-10-02T12:34:27,779 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-10-02T12:34:27,782 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:27,782 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:34:27,796 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-10-02T12:34:27,796 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-10-02T12:34:27,797 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-10-02T12:34:27,798 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-10-02T12:34:27,798 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-10-02T12:34:27,802 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:27,816 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-10-02T12:34:27,818 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-10-02T12:34:27,819 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-10-02T12:34:27,827 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1991d3c4 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:34:27,836 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510915e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:27,837 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:34:27,840 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-10-02T12:34:27,840 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-10-02T12:34:27,842 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696250067842,5,FailOnTimeoutGroup] 2023-10-02T12:34:27,843 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696250067842,5,FailOnTimeoutGroup] 2023-10-02T12:34:27,843 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:27,843 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-10-02T12:34:27,845 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:27,845 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:27,862 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,39919,1696250062021 with isa=jenkins-hbase3.apache.org/172.31.12.81:40211, startcode=1696250065770 2023-10-02T12:34:27,867 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39919 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:27,882 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3 2023-10-02T12:34:27,882 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:33727 2023-10-02T12:34:27,882 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=38657 2023-10-02T12:34:27,892 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-10-02T12:34:27,893 DEBUG [RS:0;jenkins-hbase3:40211 {}] zookeeper.ZKUtil(111): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:27,893 WARN [RS:0;jenkins-hbase3:40211 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:34:27,893 INFO [RS:0;jenkins-hbase3:40211 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:34:27,894 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:27,916 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,40211,1696250065770] 2023-10-02T12:34:27,934 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-10-02T12:34:27,951 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-10-02T12:34:27,970 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-10-02T12:34:27,978 INFO [RS:0;jenkins-hbase3:40211 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-10-02T12:34:27,979 INFO [RS:0;jenkins-hbase3:40211 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:27,980 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-10-02T12:34:27,994 INFO [RS:0;jenkins-hbase3:40211 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:27,994 DEBUG [RS:0;jenkins-hbase3:40211 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:34:27,994 DEBUG [RS:0;jenkins-hbase3:40211 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:34:27,995 DEBUG [RS:0;jenkins-hbase3:40211 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:34:27,995 DEBUG [RS:0;jenkins-hbase3:40211 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:34:27,995 DEBUG [RS:0;jenkins-hbase3:40211 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:34:27,995 DEBUG [RS:0;jenkins-hbase3:40211 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:34:27,995 DEBUG [RS:0;jenkins-hbase3:40211 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:34:27,996 DEBUG [RS:0;jenkins-hbase3:40211 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:34:27,996 DEBUG [RS:0;jenkins-hbase3:40211 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:34:27,996 DEBUG [RS:0;jenkins-hbase3:40211 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:34:27,996 DEBUG [RS:0;jenkins-hbase3:40211 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:34:27,997 DEBUG [RS:0;jenkins-hbase3:40211 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:34:27,997 DEBUG [RS:0;jenkins-hbase3:40211 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:34:28,002 INFO [RS:0;jenkins-hbase3:40211 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:28,002 INFO [RS:0;jenkins-hbase3:40211 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:28,002 INFO [RS:0;jenkins-hbase3:40211 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:28,002 INFO [RS:0;jenkins-hbase3:40211 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:28,003 INFO [RS:0;jenkins-hbase3:40211 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,40211,1696250065770-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:34:28,034 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-10-02T12:34:28,037 INFO [RS:0;jenkins-hbase3:40211 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,40211,1696250065770-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:28,058 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,40211,1696250065770 started 2023-10-02T12:34:28,059 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,40211,1696250065770, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:40211, sessionid=0x101732b40f20001 2023-10-02T12:34:28,059 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-10-02T12:34:28,060 DEBUG [RS:0;jenkins-hbase3:40211 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:28,060 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,40211,1696250065770' 2023-10-02T12:34:28,060 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-10-02T12:34:28,061 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-10-02T12:34:28,061 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-10-02T12:34:28,062 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-10-02T12:34:28,062 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,40211,1696250065770' 2023-10-02T12:34:28,062 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:34:28,064 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:34:28,065 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-10-02T12:34:28,065 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-10-02T12:34:28,065 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-10-02T12:34:28,065 DEBUG [RS:0;jenkins-hbase3:40211 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:28,065 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,40211,1696250065770' 2023-10-02T12:34:28,065 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-10-02T12:34:28,066 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-10-02T12:34:28,066 DEBUG [RS:0;jenkins-hbase3:40211 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-10-02T12:34:28,066 INFO [RS:0;jenkins-hbase3:40211 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-10-02T12:34:28,066 INFO [RS:0;jenkins-hbase3:40211 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-10-02T12:34:28,180 INFO [RS:0;jenkins-hbase3:40211 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:34:28,183 INFO [RS:0;jenkins-hbase3:40211 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C40211%2C1696250065770, suffix=, logDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770, archiveDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs, maxLogs=32 2023-10-02T12:34:28,197 DEBUG [RS:0;jenkins-hbase3:40211 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250068185, exclude list is [], retry=0 2023-10-02T12:34:28,207 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:34:28,213 INFO [RS:0;jenkins-hbase3:40211 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250068185 2023-10-02T12:34:28,218 DEBUG [RS:0;jenkins-hbase3:40211 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:34:28,219 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-10-02T12:34:28,220 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3 2023-10-02T12:34:28,642 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:28,645 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:34:28,648 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:34:28,648 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:28,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:34:28,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:34:28,651 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:34:28,651 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:28,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:34:28,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:34:28,655 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:34:28,655 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:28,656 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:34:28,657 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740 2023-10-02T12:34:28,658 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740 2023-10-02T12:34:28,661 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:34:28,664 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:34:28,668 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:34:28,669 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10592230560, jitterRate=-0.013521656394004822}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:34:28,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:34:28,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:34:28,672 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:34:28,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:34:28,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:34:28,673 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:34:28,675 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:34:28,675 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:34:28,679 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:34:28,679 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-10-02T12:34:28,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-10-02T12:34:28,711 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-10-02T12:34:28,718 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-10-02T12:34:28,871 DEBUG [jenkins-hbase3:39919 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-10-02T12:34:28,885 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:28,891 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,40211,1696250065770, state=OPENING 2023-10-02T12:34:28,904 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-10-02T12:34:28,911 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:28,911 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:28,912 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:34:28,912 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:34:28,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,40211,1696250065770}] 2023-10-02T12:34:29,117 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:29,120 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-10-02T12:34:29,125 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45622, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-10-02T12:34:29,149 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-10-02T12:34:29,151 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:34:29,152 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-10-02T12:34:29,155 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta, suffix=.meta, logDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770, archiveDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs, maxLogs=32 2023-10-02T12:34:29,172 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250069157.meta, exclude list is [], retry=0 2023-10-02T12:34:29,176 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:34:29,180 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250069157.meta 2023-10-02T12:34:29,181 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:34:29,181 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:34:29,186 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:34:29,187 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-10-02T12:34:29,208 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-10-02T12:34:29,208 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-10-02T12:34:29,214 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-10-02T12:34:29,215 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:29,215 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-10-02T12:34:29,215 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-10-02T12:34:29,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:34:29,224 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:34:29,224 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:29,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:34:29,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:34:29,227 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:34:29,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:29,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:34:29,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:34:29,230 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:34:29,230 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:29,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:34:29,232 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740 2023-10-02T12:34:29,235 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740 2023-10-02T12:34:29,239 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:34:29,243 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:34:29,245 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11105768480, jitterRate=0.03430528938770294}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:34:29,247 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:34:29,263 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1696250069107 2023-10-02T12:34:29,287 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-10-02T12:34:29,288 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-10-02T12:34:29,289 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:29,291 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,40211,1696250065770, state=OPEN 2023-10-02T12:34:29,296 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:34:29,296 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:34:29,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:34:29,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:34:29,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-10-02T12:34:29,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,40211,1696250065770 in 381 msec 2023-10-02T12:34:29,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-10-02T12:34:29,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 609 msec 2023-10-02T12:34:29,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.9440 sec 2023-10-02T12:34:29,541 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1696250069541, completionTime=-1 2023-10-02T12:34:29,542 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-10-02T12:34:29,542 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-10-02T12:34:29,639 DEBUG [hconnection-0x40922443-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:29,644 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45630, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:29,668 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-10-02T12:34:29,668 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1696250129668 2023-10-02T12:34:29,669 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1696250189669 2023-10-02T12:34:29,669 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 126 msec 2023-10-02T12:34:29,701 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39919,1696250062021-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:29,702 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39919,1696250062021-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:29,702 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39919,1696250062021-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:29,705 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:39919, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:29,706 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-10-02T12:34:29,714 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-10-02T12:34:29,736 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-10-02T12:34:29,738 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:34:29,754 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-10-02T12:34:29,759 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:34:29,762 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:29,765 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:34:29,789 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8a7643b8cd437a145a2c65d75c6a0149, NAME => 'hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3 2023-10-02T12:34:29,802 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:29,802 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 8a7643b8cd437a145a2c65d75c6a0149, disabling compactions & flushes 2023-10-02T12:34:29,803 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. 2023-10-02T12:34:29,803 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. 2023-10-02T12:34:29,803 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. after waiting 0 ms 2023-10-02T12:34:29,803 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. 2023-10-02T12:34:29,803 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. 2023-10-02T12:34:29,803 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8a7643b8cd437a145a2c65d75c6a0149: 2023-10-02T12:34:29,806 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:34:29,826 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1696250069808"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250069808"}]},"ts":"1696250069808"} 2023-10-02T12:34:29,876 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:34:29,882 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:34:29,890 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250069882"}]},"ts":"1696250069882"} 2023-10-02T12:34:29,895 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-10-02T12:34:29,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8a7643b8cd437a145a2c65d75c6a0149, ASSIGN}] 2023-10-02T12:34:29,904 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8a7643b8cd437a145a2c65d75c6a0149, ASSIGN 2023-10-02T12:34:29,907 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=8a7643b8cd437a145a2c65d75c6a0149, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40211,1696250065770; forceNewPlan=false, retain=false 2023-10-02T12:34:30,058 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8a7643b8cd437a145a2c65d75c6a0149, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:30,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 8a7643b8cd437a145a2c65d75c6a0149, server=jenkins-hbase3.apache.org,40211,1696250065770}] 2023-10-02T12:34:30,239 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. 2023-10-02T12:34:30,239 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 8a7643b8cd437a145a2c65d75c6a0149, NAME => 'hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:34:30,241 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:34:30,242 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 8a7643b8cd437a145a2c65d75c6a0149 2023-10-02T12:34:30,242 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:30,242 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 8a7643b8cd437a145a2c65d75c6a0149 2023-10-02T12:34:30,242 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 8a7643b8cd437a145a2c65d75c6a0149 2023-10-02T12:34:30,246 INFO [StoreOpener-8a7643b8cd437a145a2c65d75c6a0149-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8a7643b8cd437a145a2c65d75c6a0149 2023-10-02T12:34:30,249 INFO [StoreOpener-8a7643b8cd437a145a2c65d75c6a0149-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a7643b8cd437a145a2c65d75c6a0149 columnFamilyName info 2023-10-02T12:34:30,249 DEBUG [StoreOpener-8a7643b8cd437a145a2c65d75c6a0149-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:30,250 INFO [StoreOpener-8a7643b8cd437a145a2c65d75c6a0149-1 {}] regionserver.HStore(324): Store=8a7643b8cd437a145a2c65d75c6a0149/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:34:30,252 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/namespace/8a7643b8cd437a145a2c65d75c6a0149 2023-10-02T12:34:30,253 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/namespace/8a7643b8cd437a145a2c65d75c6a0149 2023-10-02T12:34:30,257 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 8a7643b8cd437a145a2c65d75c6a0149 2023-10-02T12:34:30,261 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/namespace/8a7643b8cd437a145a2c65d75c6a0149/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:34:30,262 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 8a7643b8cd437a145a2c65d75c6a0149; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10874631520, jitterRate=0.012778982520103455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:34:30,263 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 8a7643b8cd437a145a2c65d75c6a0149: 2023-10-02T12:34:30,266 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149., pid=6, masterSystemTime=1696250070230 2023-10-02T12:34:30,270 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. 2023-10-02T12:34:30,271 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. 2023-10-02T12:34:30,271 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8a7643b8cd437a145a2c65d75c6a0149, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:30,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-10-02T12:34:30,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 8a7643b8cd437a145a2c65d75c6a0149, server=jenkins-hbase3.apache.org,40211,1696250065770 in 203 msec 2023-10-02T12:34:30,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-10-02T12:34:30,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=8a7643b8cd437a145a2c65d75c6a0149, ASSIGN in 381 msec 2023-10-02T12:34:30,289 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:34:30,290 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250070289"}]},"ts":"1696250070289"} 2023-10-02T12:34:30,295 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-10-02T12:34:30,298 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:34:30,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 560 msec 2023-10-02T12:34:30,360 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-10-02T12:34:30,362 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-10-02T12:34:30,362 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:30,362 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:34:30,447 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-10-02T12:34:30,483 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:34:30,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 57 msec 2023-10-02T12:34:30,497 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-10-02T12:34:30,520 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:34:30,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 29 msec 2023-10-02T12:34:30,540 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-10-02T12:34:30,543 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-10-02T12:34:30,544 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.610sec 2023-10-02T12:34:30,547 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-10-02T12:34:30,549 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-10-02T12:34:30,551 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-10-02T12:34:30,554 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-10-02T12:34:30,555 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-10-02T12:34:30,557 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39919,1696250062021-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:34:30,558 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,39919,1696250062021-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-10-02T12:34:30,605 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-10-02T12:34:30,607 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-10-02T12:34:30,633 DEBUG [Listener at localhost/34813 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x10da4c22 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:34:30,684 DEBUG [Listener at localhost/34813 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625f8d4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:30,694 DEBUG [Listener at localhost/34813 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-10-02T12:34:30,695 DEBUG [Listener at localhost/34813 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-10-02T12:34:30,740 DEBUG [hconnection-0x72683eba-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:30,766 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36910, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:30,791 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,39919,1696250062021 2023-10-02T12:34:30,791 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-10-02T12:34:30,791 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/test.cache.data in system properties and HBase conf 2023-10-02T12:34:30,791 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:34:30,791 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:34:30,791 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:34:30,792 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:34:30,792 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:34:30,792 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:34:30,792 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:34:30,792 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:34:30,792 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:34:30,792 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:34:30,793 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:34:30,793 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:34:30,793 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:34:30,793 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:34:30,793 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:34:30,793 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:34:30,794 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:34:30,794 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:34:30,794 INFO [Listener at localhost/34813 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-10-02T12:34:32,675 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:34:33,173 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:34:33,183 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_41425_jobhistory____.gnsnz0/webapp 2023-10-02T12:34:33,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-10-02T12:34:33,349 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-10-02T12:34:33,356 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-10-02T12:34:33,356 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-10-02T12:34:33,359 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:34:33,359 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-10-02T12:34:33,359 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:34:33,359 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-10-02T12:34:33,359 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-10-02T12:34:33,359 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-10-02T12:34:33,397 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-10-02T12:34:33,397 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-10-02T12:34:33,397 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:34:33,402 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:34:33,502 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:34:33,621 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:34:33,942 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-10-02T12:34:33,944 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-10-02T12:34:33,988 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-10-02T12:34:34,031 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:41425 2023-10-02T12:34:34,160 WARN [Listener at jenkins-hbase3.apache.org/46631 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:34:34,320 WARN [Listener at jenkins-hbase3.apache.org/46631 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:34:34,321 INFO [Listener at jenkins-hbase3.apache.org/46631 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:34:34,334 INFO [Listener at jenkins-hbase3.apache.org/46631 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_34825_cluster____.2d6hsn/webapp 2023-10-02T12:34:34,438 INFO [Listener at jenkins-hbase3.apache.org/46631 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:34:34,439 INFO [Listener at jenkins-hbase3.apache.org/46631 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-10-02T12:34:34,439 INFO [Listener at jenkins-hbase3.apache.org/46631 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:34:34,439 INFO [Listener at jenkins-hbase3.apache.org/46631 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:34:34,450 INFO [Listener at jenkins-hbase3.apache.org/46631 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:34:34,733 INFO [Listener at jenkins-hbase3.apache.org/46631 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:34:34,787 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:34:34,939 INFO [Listener at jenkins-hbase3.apache.org/46631 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:34:34,951 INFO [Listener at jenkins-hbase3.apache.org/46631 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34825 2023-10-02T12:34:35,644 WARN [Listener at jenkins-hbase3.apache.org/41669 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:34:35,678 WARN [Listener at jenkins-hbase3.apache.org/41669 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:34:35,678 INFO [Listener at jenkins-hbase3.apache.org/41669 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:34:35,687 INFO [Listener at jenkins-hbase3.apache.org/41669 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_38661_node____.hdisc5/webapp 2023-10-02T12:34:35,764 INFO [Listener at jenkins-hbase3.apache.org/41669 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:34:35,765 INFO [Listener at jenkins-hbase3.apache.org/41669 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:34:35,765 INFO [Listener at jenkins-hbase3.apache.org/41669 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:34:35,765 INFO [Listener at jenkins-hbase3.apache.org/41669 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:34:35,777 INFO [Listener at jenkins-hbase3.apache.org/41669 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:34:35,808 INFO [Listener at jenkins-hbase3.apache.org/41669 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:34:35,962 INFO [Listener at jenkins-hbase3.apache.org/41669 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:34:35,980 INFO [Listener at jenkins-hbase3.apache.org/41669 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:38661 2023-10-02T12:34:36,308 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:34:36,329 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:34:36,330 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:34:36,340 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_43053_node____hucemm/webapp 2023-10-02T12:34:36,413 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:34:36,413 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:34:36,413 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:34:36,413 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:34:36,425 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:34:36,449 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:34:36,535 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:34:36,547 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:43053 2023-10-02T12:34:36,585 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-10-02T12:34:36,586 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:33727/backupUT 2023-10-02T12:34:36,600 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:34:36,606 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42904, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:34:36,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-10-02T12:34:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-10-02T12:34:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:34:36,662 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:34:36,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 45 msec 2023-10-02T12:34:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:34:36,755 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-10-02T12:34:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-10-02T12:34:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:34:36,772 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:34:36,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 24 msec 2023-10-02T12:34:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:34:36,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-10-02T12:34:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-10-02T12:34:36,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:34:36,894 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:34:36,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 36 msec 2023-10-02T12:34:36,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:34:36,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-10-02T12:34:36,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-10-02T12:34:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:34:37,001 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:34:37,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 23 msec 2023-10-02T12:34:37,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:34:37,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'test-1696250076586', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:34:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1696250076586 2023-10-02T12:34:37,120 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1696250076586 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:34:37,121 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1696250076586" procId is: 13 2023-10-02T12:34:37,121 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:37,122 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1696250076586 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:34:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:34:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:34:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:34:37,558 INFO [RegionOpenAndInit-test-1696250076586-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 04056acb44d8ef0206796c62cd889bed, NAME => 'test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1696250076586', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3 2023-10-02T12:34:37,582 DEBUG [RegionOpenAndInit-test-1696250076586-pool-0 {}] regionserver.HRegion(888): Instantiated test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:37,582 DEBUG [RegionOpenAndInit-test-1696250076586-pool-0 {}] regionserver.HRegion(1668): Closing 04056acb44d8ef0206796c62cd889bed, disabling compactions & flushes 2023-10-02T12:34:37,582 INFO [RegionOpenAndInit-test-1696250076586-pool-0 {}] regionserver.HRegion(1690): Closing region test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:34:37,582 DEBUG [RegionOpenAndInit-test-1696250076586-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:34:37,582 DEBUG [RegionOpenAndInit-test-1696250076586-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. after waiting 0 ms 2023-10-02T12:34:37,582 DEBUG [RegionOpenAndInit-test-1696250076586-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:34:37,582 INFO [RegionOpenAndInit-test-1696250076586-pool-0 {}] regionserver.HRegion(1909): Closed test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:34:37,582 DEBUG [RegionOpenAndInit-test-1696250076586-pool-0 {}] regionserver.HRegion(1622): Region close journal for 04056acb44d8ef0206796c62cd889bed: 2023-10-02T12:34:37,584 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1696250076586 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:34:37,585 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1696250077584"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250077584"}]},"ts":"1696250077584"} 2023-10-02T12:34:37,588 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:34:37,589 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1696250076586 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:34:37,589 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696250076586","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250077589"}]},"ts":"1696250077589"} 2023-10-02T12:34:37,592 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696250076586, state=ENABLING in hbase:meta 2023-10-02T12:34:37,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696250076586, region=04056acb44d8ef0206796c62cd889bed, ASSIGN}] 2023-10-02T12:34:37,675 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696250076586, region=04056acb44d8ef0206796c62cd889bed, ASSIGN 2023-10-02T12:34:37,676 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1696250076586, region=04056acb44d8ef0206796c62cd889bed, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40211,1696250065770; forceNewPlan=false, retain=false 2023-10-02T12:34:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:34:37,828 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=04056acb44d8ef0206796c62cd889bed, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:37,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 04056acb44d8ef0206796c62cd889bed, server=jenkins-hbase3.apache.org,40211,1696250065770}] 2023-10-02T12:34:38,000 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:34:38,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 04056acb44d8ef0206796c62cd889bed, NAME => 'test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:34:38,001 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:34:38,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1696250076586 04056acb44d8ef0206796c62cd889bed 2023-10-02T12:34:38,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:38,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 04056acb44d8ef0206796c62cd889bed 2023-10-02T12:34:38,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 04056acb44d8ef0206796c62cd889bed 2023-10-02T12:34:38,005 INFO [StoreOpener-04056acb44d8ef0206796c62cd889bed-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 04056acb44d8ef0206796c62cd889bed 2023-10-02T12:34:38,007 INFO [StoreOpener-04056acb44d8ef0206796c62cd889bed-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04056acb44d8ef0206796c62cd889bed columnFamilyName f 2023-10-02T12:34:38,008 DEBUG [StoreOpener-04056acb44d8ef0206796c62cd889bed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:38,009 INFO [StoreOpener-04056acb44d8ef0206796c62cd889bed-1 {}] regionserver.HStore(324): Store=04056acb44d8ef0206796c62cd889bed/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:34:38,011 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed 2023-10-02T12:34:38,012 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed 2023-10-02T12:34:38,016 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 04056acb44d8ef0206796c62cd889bed 2023-10-02T12:34:38,023 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:34:38,024 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 04056acb44d8ef0206796c62cd889bed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9395408160, jitterRate=-0.12498442828655243}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:34:38,025 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 04056acb44d8ef0206796c62cd889bed: 2023-10-02T12:34:38,026 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed., pid=15, masterSystemTime=1696250077985 2023-10-02T12:34:38,029 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:34:38,029 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:34:38,030 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=04056acb44d8ef0206796c62cd889bed, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:38,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-10-02T12:34:38,037 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 04056acb44d8ef0206796c62cd889bed, server=jenkins-hbase3.apache.org,40211,1696250065770 in 202 msec 2023-10-02T12:34:38,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-10-02T12:34:38,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1696250076586, region=04056acb44d8ef0206796c62cd889bed, ASSIGN in 365 msec 2023-10-02T12:34:38,042 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1696250076586 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:34:38,042 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696250076586","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250078042"}]},"ts":"1696250078042"} 2023-10-02T12:34:38,044 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696250076586, state=ENABLED in hbase:meta 2023-10-02T12:34:38,126 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1696250076586 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:34:38,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1696250076586 in 1.0110 sec 2023-10-02T12:34:38,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:34:38,239 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:test-1696250076586, procId: 13 completed 2023-10-02T12:34:38,241 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x11513b47 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:34:38,319 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5732e350, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:38,322 DEBUG [hconnection-0x77468ee0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:38,326 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36918, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:38,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40211 {}] regionserver.HRegion(8210): writing data to region test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:34:38,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:test-16962500765861', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:34:38,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16962500765861 2023-10-02T12:34:38,464 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962500765861 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:34:38,464 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:38,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16962500765861" procId is: 16 2023-10-02T12:34:38,465 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16962500765861 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:34:38,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:34:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:34:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:34:38,889 INFO [RegionOpenAndInit-ns2:test-16962500765861-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 97d4833ae9b729f41ce094e6b5f5d9e0, NAME => 'ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16962500765861', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3 2023-10-02T12:34:38,911 DEBUG [RegionOpenAndInit-ns2:test-16962500765861-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:38,911 DEBUG [RegionOpenAndInit-ns2:test-16962500765861-pool-0 {}] regionserver.HRegion(1668): Closing 97d4833ae9b729f41ce094e6b5f5d9e0, disabling compactions & flushes 2023-10-02T12:34:38,911 INFO [RegionOpenAndInit-ns2:test-16962500765861-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:34:38,911 DEBUG [RegionOpenAndInit-ns2:test-16962500765861-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:34:38,911 DEBUG [RegionOpenAndInit-ns2:test-16962500765861-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. after waiting 0 ms 2023-10-02T12:34:38,911 DEBUG [RegionOpenAndInit-ns2:test-16962500765861-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:34:38,911 INFO [RegionOpenAndInit-ns2:test-16962500765861-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:34:38,911 DEBUG [RegionOpenAndInit-ns2:test-16962500765861-pool-0 {}] regionserver.HRegion(1622): Region close journal for 97d4833ae9b729f41ce094e6b5f5d9e0: 2023-10-02T12:34:38,913 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16962500765861 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:34:38,913 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696250078913"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250078913"}]},"ts":"1696250078913"} 2023-10-02T12:34:38,916 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:34:38,918 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16962500765861 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:34:38,918 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962500765861","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250078918"}]},"ts":"1696250078918"} 2023-10-02T12:34:38,920 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962500765861, state=ENABLING in hbase:meta 2023-10-02T12:34:38,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962500765861, region=97d4833ae9b729f41ce094e6b5f5d9e0, ASSIGN}] 2023-10-02T12:34:38,927 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962500765861, region=97d4833ae9b729f41ce094e6b5f5d9e0, ASSIGN 2023-10-02T12:34:38,929 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16962500765861, region=97d4833ae9b729f41ce094e6b5f5d9e0, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40211,1696250065770; forceNewPlan=false, retain=false 2023-10-02T12:34:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:34:39,080 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=97d4833ae9b729f41ce094e6b5f5d9e0, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:39,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 97d4833ae9b729f41ce094e6b5f5d9e0, server=jenkins-hbase3.apache.org,40211,1696250065770}] 2023-10-02T12:34:39,241 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:34:39,241 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 97d4833ae9b729f41ce094e6b5f5d9e0, NAME => 'ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:34:39,242 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:34:39,242 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962500765861 97d4833ae9b729f41ce094e6b5f5d9e0 2023-10-02T12:34:39,242 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:39,242 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 97d4833ae9b729f41ce094e6b5f5d9e0 2023-10-02T12:34:39,242 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 97d4833ae9b729f41ce094e6b5f5d9e0 2023-10-02T12:34:39,244 INFO [StoreOpener-97d4833ae9b729f41ce094e6b5f5d9e0-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 97d4833ae9b729f41ce094e6b5f5d9e0 2023-10-02T12:34:39,246 INFO [StoreOpener-97d4833ae9b729f41ce094e6b5f5d9e0-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97d4833ae9b729f41ce094e6b5f5d9e0 columnFamilyName f 2023-10-02T12:34:39,246 DEBUG [StoreOpener-97d4833ae9b729f41ce094e6b5f5d9e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:39,247 INFO [StoreOpener-97d4833ae9b729f41ce094e6b5f5d9e0-1 {}] regionserver.HStore(324): Store=97d4833ae9b729f41ce094e6b5f5d9e0/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:34:39,248 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0 2023-10-02T12:34:39,249 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0 2023-10-02T12:34:39,253 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 97d4833ae9b729f41ce094e6b5f5d9e0 2023-10-02T12:34:39,262 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:34:39,263 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 97d4833ae9b729f41ce094e6b5f5d9e0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9524037920, jitterRate=-0.11300484836101532}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:34:39,274 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 97d4833ae9b729f41ce094e6b5f5d9e0: 2023-10-02T12:34:39,276 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0., pid=18, masterSystemTime=1696250079236 2023-10-02T12:34:39,279 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:34:39,279 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:34:39,280 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=97d4833ae9b729f41ce094e6b5f5d9e0, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:39,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-10-02T12:34:39,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 97d4833ae9b729f41ce094e6b5f5d9e0, server=jenkins-hbase3.apache.org,40211,1696250065770 in 201 msec 2023-10-02T12:34:39,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-10-02T12:34:39,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16962500765861, region=97d4833ae9b729f41ce094e6b5f5d9e0, ASSIGN in 363 msec 2023-10-02T12:34:39,292 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16962500765861 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:34:39,292 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962500765861","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250079292"}]},"ts":"1696250079292"} 2023-10-02T12:34:39,294 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962500765861, state=ENABLED in hbase:meta 2023-10-02T12:34:39,297 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962500765861 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:34:39,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16962500765861 in 837 msec 2023-10-02T12:34:39,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:34:39,577 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns2:test-16962500765861, procId: 16 completed 2023-10-02T12:34:39,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40211 {}] regionserver.HRegion(8210): writing data to region ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:34:39,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:test-16962500765862', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:34:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16962500765862 2023-10-02T12:34:39,702 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962500765862 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:34:39,702 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:39,703 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16962500765862" procId is: 19 2023-10-02T12:34:39,704 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16962500765862 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:34:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:34:39,719 INFO [RegionOpenAndInit-ns3:test-16962500765862-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 66825b0933bd359e9ecffdc9dc0cef0e, NAME => 'ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16962500765862', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3 2023-10-02T12:34:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:34:39,946 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1696250076586' 2023-10-02T12:34:39,946 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16962500765861' 2023-10-02T12:34:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:34:40,144 DEBUG [RegionOpenAndInit-ns3:test-16962500765862-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:40,145 DEBUG [RegionOpenAndInit-ns3:test-16962500765862-pool-0 {}] regionserver.HRegion(1668): Closing 66825b0933bd359e9ecffdc9dc0cef0e, disabling compactions & flushes 2023-10-02T12:34:40,145 INFO [RegionOpenAndInit-ns3:test-16962500765862-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:34:40,145 DEBUG [RegionOpenAndInit-ns3:test-16962500765862-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:34:40,145 DEBUG [RegionOpenAndInit-ns3:test-16962500765862-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. after waiting 0 ms 2023-10-02T12:34:40,145 DEBUG [RegionOpenAndInit-ns3:test-16962500765862-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:34:40,145 INFO [RegionOpenAndInit-ns3:test-16962500765862-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:34:40,145 DEBUG [RegionOpenAndInit-ns3:test-16962500765862-pool-0 {}] regionserver.HRegion(1622): Region close journal for 66825b0933bd359e9ecffdc9dc0cef0e: 2023-10-02T12:34:40,147 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16962500765862 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:34:40,147 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696250080147"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250080147"}]},"ts":"1696250080147"} 2023-10-02T12:34:40,150 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:34:40,152 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16962500765862 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:34:40,152 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962500765862","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250080152"}]},"ts":"1696250080152"} 2023-10-02T12:34:40,154 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962500765862, state=ENABLING in hbase:meta 2023-10-02T12:34:40,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962500765862, region=66825b0933bd359e9ecffdc9dc0cef0e, ASSIGN}] 2023-10-02T12:34:40,165 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962500765862, region=66825b0933bd359e9ecffdc9dc0cef0e, ASSIGN 2023-10-02T12:34:40,167 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16962500765862, region=66825b0933bd359e9ecffdc9dc0cef0e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40211,1696250065770; forceNewPlan=false, retain=false 2023-10-02T12:34:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:34:40,318 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=66825b0933bd359e9ecffdc9dc0cef0e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:40,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 66825b0933bd359e9ecffdc9dc0cef0e, server=jenkins-hbase3.apache.org,40211,1696250065770}] 2023-10-02T12:34:40,480 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:34:40,481 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 66825b0933bd359e9ecffdc9dc0cef0e, NAME => 'ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:34:40,481 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:34:40,481 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962500765862 66825b0933bd359e9ecffdc9dc0cef0e 2023-10-02T12:34:40,481 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:40,482 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 66825b0933bd359e9ecffdc9dc0cef0e 2023-10-02T12:34:40,482 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 66825b0933bd359e9ecffdc9dc0cef0e 2023-10-02T12:34:40,484 INFO [StoreOpener-66825b0933bd359e9ecffdc9dc0cef0e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 66825b0933bd359e9ecffdc9dc0cef0e 2023-10-02T12:34:40,486 INFO [StoreOpener-66825b0933bd359e9ecffdc9dc0cef0e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66825b0933bd359e9ecffdc9dc0cef0e columnFamilyName f 2023-10-02T12:34:40,486 DEBUG [StoreOpener-66825b0933bd359e9ecffdc9dc0cef0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:40,487 INFO [StoreOpener-66825b0933bd359e9ecffdc9dc0cef0e-1 {}] regionserver.HStore(324): Store=66825b0933bd359e9ecffdc9dc0cef0e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:34:40,488 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns3/test-16962500765862/66825b0933bd359e9ecffdc9dc0cef0e 2023-10-02T12:34:40,489 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns3/test-16962500765862/66825b0933bd359e9ecffdc9dc0cef0e 2023-10-02T12:34:40,492 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 66825b0933bd359e9ecffdc9dc0cef0e 2023-10-02T12:34:40,500 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns3/test-16962500765862/66825b0933bd359e9ecffdc9dc0cef0e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:34:40,501 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 66825b0933bd359e9ecffdc9dc0cef0e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11779619520, jitterRate=0.09706255793571472}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:34:40,502 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 66825b0933bd359e9ecffdc9dc0cef0e: 2023-10-02T12:34:40,503 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e., pid=21, masterSystemTime=1696250080475 2023-10-02T12:34:40,506 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:34:40,506 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:34:40,507 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=66825b0933bd359e9ecffdc9dc0cef0e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:40,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-10-02T12:34:40,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 66825b0933bd359e9ecffdc9dc0cef0e, server=jenkins-hbase3.apache.org,40211,1696250065770 in 188 msec 2023-10-02T12:34:40,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-10-02T12:34:40,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16962500765862, region=66825b0933bd359e9ecffdc9dc0cef0e, ASSIGN in 350 msec 2023-10-02T12:34:40,517 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16962500765862 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:34:40,517 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962500765862","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250080517"}]},"ts":"1696250080517"} 2023-10-02T12:34:40,519 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962500765862, state=ENABLED in hbase:meta 2023-10-02T12:34:40,522 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962500765862 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:34:40,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16962500765862 in 825 msec 2023-10-02T12:34:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:34:40,814 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns3:test-16962500765862, procId: 19 completed 2023-10-02T12:34:40,814 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16962500765862 get assigned. Timeout = 60000ms 2023-10-02T12:34:40,816 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:34:40,821 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16962500765862 assigned to meta. Checking AM states. 2023-10-02T12:34:40,822 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:34:40,822 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16962500765862 assigned. 2023-10-02T12:34:40,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns4:test-16962500765863', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:34:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16962500765863 2023-10-02T12:34:40,828 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962500765863 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:34:40,829 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:40,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16962500765863" procId is: 22 2023-10-02T12:34:40,830 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16962500765863 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:34:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:34:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:34:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:34:41,247 INFO [RegionOpenAndInit-ns4:test-16962500765863-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => ee1a09a86970bc5e288b8a3080cb4647, NAME => 'ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16962500765863', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3 2023-10-02T12:34:41,261 DEBUG [RegionOpenAndInit-ns4:test-16962500765863-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:41,261 DEBUG [RegionOpenAndInit-ns4:test-16962500765863-pool-0 {}] regionserver.HRegion(1668): Closing ee1a09a86970bc5e288b8a3080cb4647, disabling compactions & flushes 2023-10-02T12:34:41,261 INFO [RegionOpenAndInit-ns4:test-16962500765863-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. 2023-10-02T12:34:41,261 DEBUG [RegionOpenAndInit-ns4:test-16962500765863-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. 2023-10-02T12:34:41,261 DEBUG [RegionOpenAndInit-ns4:test-16962500765863-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. after waiting 0 ms 2023-10-02T12:34:41,261 DEBUG [RegionOpenAndInit-ns4:test-16962500765863-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. 2023-10-02T12:34:41,261 INFO [RegionOpenAndInit-ns4:test-16962500765863-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. 2023-10-02T12:34:41,261 DEBUG [RegionOpenAndInit-ns4:test-16962500765863-pool-0 {}] regionserver.HRegion(1622): Region close journal for ee1a09a86970bc5e288b8a3080cb4647: 2023-10-02T12:34:41,263 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16962500765863 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:34:41,263 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696250081263"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250081263"}]},"ts":"1696250081263"} 2023-10-02T12:34:41,265 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:34:41,266 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16962500765863 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:34:41,267 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962500765863","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250081266"}]},"ts":"1696250081266"} 2023-10-02T12:34:41,269 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962500765863, state=ENABLING in hbase:meta 2023-10-02T12:34:41,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962500765863, region=ee1a09a86970bc5e288b8a3080cb4647, ASSIGN}] 2023-10-02T12:34:41,276 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962500765863, region=ee1a09a86970bc5e288b8a3080cb4647, ASSIGN 2023-10-02T12:34:41,277 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16962500765863, region=ee1a09a86970bc5e288b8a3080cb4647, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40211,1696250065770; forceNewPlan=false, retain=false 2023-10-02T12:34:41,429 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=ee1a09a86970bc5e288b8a3080cb4647, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:41,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure ee1a09a86970bc5e288b8a3080cb4647, server=jenkins-hbase3.apache.org,40211,1696250065770}] 2023-10-02T12:34:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:34:41,589 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. 2023-10-02T12:34:41,590 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => ee1a09a86970bc5e288b8a3080cb4647, NAME => 'ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:34:41,590 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:34:41,590 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962500765863 ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:34:41,590 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:41,590 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:34:41,590 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:34:41,592 INFO [StoreOpener-ee1a09a86970bc5e288b8a3080cb4647-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:34:41,594 INFO [StoreOpener-ee1a09a86970bc5e288b8a3080cb4647-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee1a09a86970bc5e288b8a3080cb4647 columnFamilyName f 2023-10-02T12:34:41,594 DEBUG [StoreOpener-ee1a09a86970bc5e288b8a3080cb4647-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:41,595 INFO [StoreOpener-ee1a09a86970bc5e288b8a3080cb4647-1 {}] regionserver.HStore(324): Store=ee1a09a86970bc5e288b8a3080cb4647/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:34:41,596 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns4/test-16962500765863/ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:34:41,597 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns4/test-16962500765863/ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:34:41,599 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:34:41,603 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns4/test-16962500765863/ee1a09a86970bc5e288b8a3080cb4647/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:34:41,603 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened ee1a09a86970bc5e288b8a3080cb4647; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11050614080, jitterRate=0.029168635606765747}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:34:41,604 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for ee1a09a86970bc5e288b8a3080cb4647: 2023-10-02T12:34:41,605 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647., pid=24, masterSystemTime=1696250081585 2023-10-02T12:34:41,608 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. 2023-10-02T12:34:41,608 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. 2023-10-02T12:34:41,609 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=ee1a09a86970bc5e288b8a3080cb4647, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:41,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-10-02T12:34:41,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure ee1a09a86970bc5e288b8a3080cb4647, server=jenkins-hbase3.apache.org,40211,1696250065770 in 179 msec 2023-10-02T12:34:41,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-10-02T12:34:41,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16962500765863, region=ee1a09a86970bc5e288b8a3080cb4647, ASSIGN in 339 msec 2023-10-02T12:34:41,617 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16962500765863 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:34:41,617 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962500765863","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250081617"}]},"ts":"1696250081617"} 2023-10-02T12:34:41,619 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962500765863, state=ENABLED in hbase:meta 2023-10-02T12:34:41,622 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962500765863 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:34:41,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16962500765863 in 798 msec 2023-10-02T12:34:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:34:41,939 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns4:test-16962500765863, procId: 22 completed 2023-10-02T12:34:41,939 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16962500765863 get assigned. Timeout = 60000ms 2023-10-02T12:34:41,939 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:34:41,943 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16962500765863 assigned to meta. Checking AM states. 2023-10-02T12:34:41,943 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:34:41,943 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16962500765863 assigned. 2023-10-02T12:34:41,944 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x11513b47 to 127.0.0.1:64600 2023-10-02T12:34:41,944 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:41,985 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.ResourceChecker(147): before: backup.TestBackupMultipleDeletes#testBackupMultipleDeletes Thread=584, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=558, ProcessCount=181, AvailableMemoryMB=3175 2023-10-02T12:34:41,985 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.ResourceChecker(130): Thread=584 is superior to 500 2023-10-02T12:34:41,988 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] backup.TestBackupMultipleDeletes(60): create full backup image for all tables 2023-10-02T12:34:41,989 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0842c64b to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:34:41,994 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4581282a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:42,004 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:42,005 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54460, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:42,019 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:34:42,021 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57800, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:34:42,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-10-02T12:34:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-10-02T12:34:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:34:42,061 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:34:42,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 33 msec 2023-10-02T12:34:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:34:42,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:34:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-10-02T12:34:42,167 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:34:42,167 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:42,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-10-02T12:34:42,168 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:34:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:34:42,193 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 5bf5964d15ce023a31881482c13240a2, NAME => 'backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3 2023-10-02T12:34:42,212 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:42,212 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 5bf5964d15ce023a31881482c13240a2, disabling compactions & flushes 2023-10-02T12:34:42,212 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:34:42,212 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:34:42,212 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. after waiting 0 ms 2023-10-02T12:34:42,212 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:34:42,212 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:34:42,213 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 5bf5964d15ce023a31881482c13240a2: 2023-10-02T12:34:42,215 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:34:42,215 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696250082215"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250082215"}]},"ts":"1696250082215"} 2023-10-02T12:34:42,217 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:34:42,219 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:34:42,219 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250082219"}]},"ts":"1696250082219"} 2023-10-02T12:34:42,221 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:34:42,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=5bf5964d15ce023a31881482c13240a2, ASSIGN}] 2023-10-02T12:34:42,229 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=5bf5964d15ce023a31881482c13240a2, ASSIGN 2023-10-02T12:34:42,230 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=5bf5964d15ce023a31881482c13240a2, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40211,1696250065770; forceNewPlan=false, retain=false 2023-10-02T12:34:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:34:42,382 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=5bf5964d15ce023a31881482c13240a2, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:42,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 5bf5964d15ce023a31881482c13240a2, server=jenkins-hbase3.apache.org,40211,1696250065770}] 2023-10-02T12:34:42,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:34:42,544 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:34:42,544 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 5bf5964d15ce023a31881482c13240a2, NAME => 'backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:34:42,545 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:34:42,545 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:42,545 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:42,545 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:42,545 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:42,547 INFO [StoreOpener-5bf5964d15ce023a31881482c13240a2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:42,551 INFO [StoreOpener-5bf5964d15ce023a31881482c13240a2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bf5964d15ce023a31881482c13240a2 columnFamilyName meta 2023-10-02T12:34:42,551 DEBUG [StoreOpener-5bf5964d15ce023a31881482c13240a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:42,552 INFO [StoreOpener-5bf5964d15ce023a31881482c13240a2-1 {}] regionserver.HStore(324): Store=5bf5964d15ce023a31881482c13240a2/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:34:42,552 INFO [StoreOpener-5bf5964d15ce023a31881482c13240a2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:42,554 INFO [StoreOpener-5bf5964d15ce023a31881482c13240a2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bf5964d15ce023a31881482c13240a2 columnFamilyName session 2023-10-02T12:34:42,555 DEBUG [StoreOpener-5bf5964d15ce023a31881482c13240a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:42,557 INFO [StoreOpener-5bf5964d15ce023a31881482c13240a2-1 {}] regionserver.HStore(324): Store=5bf5964d15ce023a31881482c13240a2/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:34:42,558 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:42,559 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:42,561 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:34:42,563 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:42,568 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:34:42,569 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 5bf5964d15ce023a31881482c13240a2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10807788320, jitterRate=0.006553724408149719}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:34:42,570 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 5bf5964d15ce023a31881482c13240a2: 2023-10-02T12:34:42,571 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2., pid=28, masterSystemTime=1696250082539 2023-10-02T12:34:42,574 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:34:42,574 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:34:42,575 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=5bf5964d15ce023a31881482c13240a2, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:42,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-10-02T12:34:42,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 5bf5964d15ce023a31881482c13240a2, server=jenkins-hbase3.apache.org,40211,1696250065770 in 191 msec 2023-10-02T12:34:42,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-10-02T12:34:42,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=5bf5964d15ce023a31881482c13240a2, ASSIGN in 353 msec 2023-10-02T12:34:42,584 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:34:42,585 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250082585"}]},"ts":"1696250082585"} 2023-10-02T12:34:42,588 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:34:42,591 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:34:42,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 429 msec 2023-10-02T12:34:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:34:42,775 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-10-02T12:34:42,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:34:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-10-02T12:34:42,800 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:34:42,800 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:42,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-10-02T12:34:42,801 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:34:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:34:42,814 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => f846b5ddba88af6ce438c513542c180e, NAME => 'backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3 2023-10-02T12:34:42,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:34:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:34:43,237 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:43,237 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing f846b5ddba88af6ce438c513542c180e, disabling compactions & flushes 2023-10-02T12:34:43,237 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. 2023-10-02T12:34:43,237 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. 2023-10-02T12:34:43,237 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. after waiting 0 ms 2023-10-02T12:34:43,238 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. 2023-10-02T12:34:43,238 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. 2023-10-02T12:34:43,238 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for f846b5ddba88af6ce438c513542c180e: 2023-10-02T12:34:43,239 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:34:43,240 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1696250083239"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250083239"}]},"ts":"1696250083239"} 2023-10-02T12:34:43,242 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:34:43,243 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:34:43,244 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250083243"}]},"ts":"1696250083243"} 2023-10-02T12:34:43,245 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-10-02T12:34:43,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=f846b5ddba88af6ce438c513542c180e, ASSIGN}] 2023-10-02T12:34:43,253 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=f846b5ddba88af6ce438c513542c180e, ASSIGN 2023-10-02T12:34:43,254 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=f846b5ddba88af6ce438c513542c180e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40211,1696250065770; forceNewPlan=false, retain=false 2023-10-02T12:34:43,406 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=f846b5ddba88af6ce438c513542c180e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:34:43,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure f846b5ddba88af6ce438c513542c180e, server=jenkins-hbase3.apache.org,40211,1696250065770}] 2023-10-02T12:34:43,518 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:34:43,579 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. 2023-10-02T12:34:43,579 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => f846b5ddba88af6ce438c513542c180e, NAME => 'backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:34:43,580 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:34:43,581 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk f846b5ddba88af6ce438c513542c180e 2023-10-02T12:34:43,581 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:34:43,581 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for f846b5ddba88af6ce438c513542c180e 2023-10-02T12:34:43,581 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for f846b5ddba88af6ce438c513542c180e 2023-10-02T12:34:43,584 INFO [StoreOpener-f846b5ddba88af6ce438c513542c180e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region f846b5ddba88af6ce438c513542c180e 2023-10-02T12:34:43,592 INFO [StoreOpener-f846b5ddba88af6ce438c513542c180e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f846b5ddba88af6ce438c513542c180e columnFamilyName meta 2023-10-02T12:34:43,595 DEBUG [StoreOpener-f846b5ddba88af6ce438c513542c180e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:43,596 INFO [StoreOpener-f846b5ddba88af6ce438c513542c180e-1 {}] regionserver.HStore(324): Store=f846b5ddba88af6ce438c513542c180e/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:34:43,596 INFO [StoreOpener-f846b5ddba88af6ce438c513542c180e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region f846b5ddba88af6ce438c513542c180e 2023-10-02T12:34:43,598 INFO [StoreOpener-f846b5ddba88af6ce438c513542c180e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f846b5ddba88af6ce438c513542c180e columnFamilyName session 2023-10-02T12:34:43,598 DEBUG [StoreOpener-f846b5ddba88af6ce438c513542c180e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:34:43,598 INFO [StoreOpener-f846b5ddba88af6ce438c513542c180e-1 {}] regionserver.HStore(324): Store=f846b5ddba88af6ce438c513542c180e/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:34:43,602 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system_bulk/f846b5ddba88af6ce438c513542c180e 2023-10-02T12:34:43,602 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system_bulk/f846b5ddba88af6ce438c513542c180e 2023-10-02T12:34:43,604 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:34:43,606 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for f846b5ddba88af6ce438c513542c180e 2023-10-02T12:34:43,611 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system_bulk/f846b5ddba88af6ce438c513542c180e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:34:43,612 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened f846b5ddba88af6ce438c513542c180e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10460514400, jitterRate=-0.025788679718971252}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:34:43,613 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for f846b5ddba88af6ce438c513542c180e: 2023-10-02T12:34:43,614 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e., pid=31, masterSystemTime=1696250083563 2023-10-02T12:34:43,620 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. 2023-10-02T12:34:43,620 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. 2023-10-02T12:34:43,621 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=f846b5ddba88af6ce438c513542c180e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:43,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-10-02T12:34:43,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure f846b5ddba88af6ce438c513542c180e, server=jenkins-hbase3.apache.org,40211,1696250065770 in 216 msec 2023-10-02T12:34:43,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-10-02T12:34:43,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=f846b5ddba88af6ce438c513542c180e, ASSIGN in 380 msec 2023-10-02T12:34:43,637 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:34:43,638 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250083637"}]},"ts":"1696250083637"} 2023-10-02T12:34:43,639 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:34:43,639 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-10-02T12:34:43,640 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-10-02T12:34:43,640 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962500765861 2023-10-02T12:34:43,640 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962500765861 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:34:43,641 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1696250076586 2023-10-02T12:34:43,641 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1696250076586 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:34:43,642 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-10-02T12:34:43,642 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-10-02T12:34:43,643 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962500765863 2023-10-02T12:34:43,643 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962500765863 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:34:43,644 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962500765862 2023-10-02T12:34:43,644 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962500765862 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:34:43,698 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:34:43,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 902 msec 2023-10-02T12:34:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:34:43,910 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-10-02T12:34:43,925 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:34:43,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:34:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250083980 (current time:1696250083980). 2023-10-02T12:34:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:34:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:34:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5d18a5a1 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:34:44,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60a8c7a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:44,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:44,052 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54466, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5d18a5a1 to 127.0.0.1:64600 2023-10-02T12:34:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:34:44,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:34:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:34:44,079 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:34:44,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:34:44,086 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:34:44,104 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot, current working dir path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:34:44,107 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:34:44,118 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:34:44,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2}] 2023-10-02T12:34:44,131 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:34:44,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40211 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-10-02T12:34:44,294 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:34:44,296 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 5bf5964d15ce023a31881482c13240a2 2/2 column families, dataSize=45 B heapSize=632 B 2023-10-02T12:34:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:34:44,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/fcaa57170a1440eb907a0c18a525ad10 is 49, key is activesession:/session:c/1696250083964/Put/seqid=0 2023-10-02T12:34:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:34:44,843 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/fcaa57170a1440eb907a0c18a525ad10 2023-10-02T12:34:44,940 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/fcaa57170a1440eb907a0c18a525ad10 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10 2023-10-02T12:34:44,953 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10, entries=1, sequenceid=5, filesize=4.9 K 2023-10-02T12:34:44,958 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 5bf5964d15ce023a31881482c13240a2 in 662ms, sequenceid=5, compaction requested=false 2023-10-02T12:34:44,958 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-10-02T12:34:44,961 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 5bf5964d15ce023a31881482c13240a2: 2023-10-02T12:34:44,961 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. for snapshot_backup_system completed. 2023-10-02T12:34:44,962 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:34:44,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:34:44,978 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:34:44,979 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10] hfiles 2023-10-02T12:34:44,979 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10 for snapshot=snapshot_backup_system 2023-10-02T12:34:44,999 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:34:45,003 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-10-02T12:34:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster(3995): Remote procedure done, pid=33 2023-10-02T12:34:45,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:45,008 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:45,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-10-02T12:34:45,013 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:34:45,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 in 888 msec 2023-10-02T12:34:45,016 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:34:45,020 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:34:45,020 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:34:45,024 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:34:45,103 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:34:45,124 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:34:45,125 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:34:45,128 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:34:45,128 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:34:45,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.0630 sec 2023-10-02T12:34:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:34:45,191 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-10-02T12:34:45,191 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(115): Backup backup_1696250081997 started at 1696250085191. 2023-10-02T12:34:45,224 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(119): Backup session backup_1696250081997 has been started. 2023-10-02T12:34:45,228 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:34:45,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:34:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:34:45,253 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:34:45,253 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:34:45,254 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:34:45,254 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:34:45,255 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:34:45,255 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:34:45,256 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:34:45,256 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,256 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:34:45,256 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:34:45,257 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,257 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:34:45,257 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:34:45,257 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:34:45,258 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:34:45,258 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:34:45,258 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:34:45,262 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:34:45,262 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:34:45,262 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:34:45,262 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:34:45,264 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:34:45,264 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:34:45,264 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:34:45,267 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,267 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:45,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,268 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:34:45,268 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:34:45,268 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:34:45,268 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:34:45,268 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:34:45,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:34:45,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:45,270 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,270 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:34:45,271 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:34:45,271 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:34:45,271 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@54f400e2[Count = 0] remaining members to acquire global barrier 2023-10-02T12:34:45,271 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:45,273 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:45,273 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:45,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:45,274 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:34:45,274 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,274 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:34:45,275 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:34:45,275 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,275 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250068185 highest: 1696250068185 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,275 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770:(num 1696250068185) roll requested 2023-10-02T12:34:45,293 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250085275, exclude list is [], retry=0 2023-10-02T12:34:45,301 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:34:45,306 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250068185 with entries=19, filesize=4.02 KB; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250085275 2023-10-02T12:34:45,307 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:34:45,307 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250068185 is not closed yet, will try archiving it next time 2023-10-02T12:34:45,307 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta:.meta(num 1696250069157) roll requested 2023-10-02T12:34:45,320 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250085307.meta, exclude list is [], retry=0 2023-10-02T12:34:45,324 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:34:45,329 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250069157.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250085307.meta 2023-10-02T12:34:45,329 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:34:45,329 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250069157.meta is not closed yet, will try archiving it next time 2023-10-02T12:34:45,337 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 62 2023-10-02T12:34:45,337 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250085275 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,338 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:34:45,340 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57802, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:34:45,359 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:34:45,359 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40211,1696250065770' in zk 2023-10-02T12:34:45,362 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,362 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:34:45,362 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:34:45,362 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,362 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:34:45,362 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:34:45,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:34:45,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:34:45,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:34:45,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:45,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,366 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:34:45,366 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:45,367 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,368 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40211,1696250065770': 2023-10-02T12:34:45,368 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:34:45,368 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:34:45,368 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:34:45,368 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:34:45,368 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:34:45,368 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:34:45,371 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:34:45,371 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:34:45,371 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:34:45,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:34:45,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:34:45,371 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:34:45,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:34:45,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:34:45,372 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:34:45,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:34:45,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:34:45,372 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:45,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:34:45,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:34:45,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:45,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,374 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:34:45,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:45,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,386 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,387 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:34:45,387 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:34:45,387 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:34:45,387 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:45,387 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:34:45,387 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:45,387 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:34:45,387 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:45,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:34:45,387 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:34:45,387 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:34:45,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:34:45,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:34:45,388 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:34:45,388 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:34:45,393 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-10-02T12:34:45,393 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:34:45,493 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:34:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:34:45,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 } 2023-10-02T12:34:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250085507 (current time:1696250085508). 2023-10-02T12:34:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:34:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696250085506_default_test-1696250076586 VERSION not specified, setting to 2 2023-10-02T12:34:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x34b99c20 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:34:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ad9e1d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:45,523 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54472, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x34b99c20 to 127.0.0.1:64600 2023-10-02T12:34:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:34:45,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 } 2023-10-02T12:34:45,529 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:34:45,530 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:34:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-10-02T12:34:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:34:45,535 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot, current working dir path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_1696250085506_default_test-1696250076586 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:34:45,537 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:34:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:34:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:34:45,948 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-10-02T12:34:45,948 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16962500765863' 2023-10-02T12:34:45,949 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16962500765862' 2023-10-02T12:34:45,985 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:34:45,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 04056acb44d8ef0206796c62cd889bed}] 2023-10-02T12:34:45,988 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 04056acb44d8ef0206796c62cd889bed 2023-10-02T12:34:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:34:46,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40211 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-10-02T12:34:46,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:34:46,142 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 04056acb44d8ef0206796c62cd889bed 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:34:46,163 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/.tmp/f/a7719d605238452e97f183af22877b79 is 37, key is row10/f:q1/1696250078345/Put/seqid=0 2023-10-02T12:34:46,573 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/.tmp/f/a7719d605238452e97f183af22877b79 2023-10-02T12:34:46,587 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/.tmp/f/a7719d605238452e97f183af22877b79 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/f/a7719d605238452e97f183af22877b79 2023-10-02T12:34:46,600 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/f/a7719d605238452e97f183af22877b79, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:34:46,602 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 04056acb44d8ef0206796c62cd889bed in 460ms, sequenceid=103, compaction requested=false 2023-10-02T12:34:46,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 04056acb44d8ef0206796c62cd889bed: 2023-10-02T12:34:46,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. for snapshot_1696250085506_default_test-1696250076586 completed. 2023-10-02T12:34:46,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed.' region-info for snapshot=snapshot_1696250085506_default_test-1696250076586 2023-10-02T12:34:46,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:34:46,604 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/f/a7719d605238452e97f183af22877b79] hfiles 2023-10-02T12:34:46,604 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/f/a7719d605238452e97f183af22877b79 for snapshot=snapshot_1696250085506_default_test-1696250076586 2023-10-02T12:34:46,619 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:34:46,619 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-10-02T12:34:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster(3995): Remote procedure done, pid=35 2023-10-02T12:34:46,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696250085506_default_test-1696250076586 on region 04056acb44d8ef0206796c62cd889bed 2023-10-02T12:34:46,620 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 04056acb44d8ef0206796c62cd889bed 2023-10-02T12:34:46,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-10-02T12:34:46,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 04056acb44d8ef0206796c62cd889bed in 637 msec 2023-10-02T12:34:46,624 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:34:46,626 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:34:46,627 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:34:46,627 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696250085506_default_test-1696250076586 2023-10-02T12:34:46,628 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_1696250085506_default_test-1696250076586 2023-10-02T12:34:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:34:46,652 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:34:46,665 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:34:46,666 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_1696250085506_default_test-1696250076586 to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_1696250085506_default_test-1696250076586 2023-10-02T12:34:46,668 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:34:46,668 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-10-02T12:34:46,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696250085506_default_test-1696250076586 table=test-1696250076586 type=FLUSH ttl=0 } in 1.1430 sec 2023-10-02T12:34:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:34:47,647 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: default:test-1696250076586, procId: 34 completed 2023-10-02T12:34:47,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 } 2023-10-02T12:34:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250087648 (current time:1696250087648). 2023-10-02T12:34:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:34:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696250087647_ns2_test-16962500765861 VERSION not specified, setting to 2 2023-10-02T12:34:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x08af7c48 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:34:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708fb9e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:47,660 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54480, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x08af7c48 to 127.0.0.1:64600 2023-10-02T12:34:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:34:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 } 2023-10-02T12:34:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-10-02T12:34:47,666 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:34:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:34:47,667 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:34:47,671 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot, current working dir path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_1696250087647_ns2_test-16962500765861 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:34:47,674 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:34:47,688 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:34:47,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 97d4833ae9b729f41ce094e6b5f5d9e0}] 2023-10-02T12:34:47,690 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 97d4833ae9b729f41ce094e6b5f5d9e0 2023-10-02T12:34:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:34:47,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40211 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-10-02T12:34:47,843 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:34:47,844 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2819): Flushing 97d4833ae9b729f41ce094e6b5f5d9e0 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:34:47,862 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/.tmp/f/4b0589f8b30b4e1cb1ee0b3f2d8f21d5 is 37, key is row10/f:q1/1696250079598/Put/seqid=0 2023-10-02T12:34:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:34:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:34:48,274 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/.tmp/f/4b0589f8b30b4e1cb1ee0b3f2d8f21d5 2023-10-02T12:34:48,283 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/.tmp/f/4b0589f8b30b4e1cb1ee0b3f2d8f21d5 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/f/4b0589f8b30b4e1cb1ee0b3f2d8f21d5 2023-10-02T12:34:48,291 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/f/4b0589f8b30b4e1cb1ee0b3f2d8f21d5, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:34:48,293 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 97d4833ae9b729f41ce094e6b5f5d9e0 in 448ms, sequenceid=103, compaction requested=false 2023-10-02T12:34:48,294 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 97d4833ae9b729f41ce094e6b5f5d9e0: 2023-10-02T12:34:48,294 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. for snapshot_1696250087647_ns2_test-16962500765861 completed. 2023-10-02T12:34:48,295 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0.' region-info for snapshot=snapshot_1696250087647_ns2_test-16962500765861 2023-10-02T12:34:48,295 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:34:48,295 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/f/4b0589f8b30b4e1cb1ee0b3f2d8f21d5] hfiles 2023-10-02T12:34:48,295 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/f/4b0589f8b30b4e1cb1ee0b3f2d8f21d5 for snapshot=snapshot_1696250087647_ns2_test-16962500765861 2023-10-02T12:34:48,705 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:34:48,705 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-10-02T12:34:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster(3995): Remote procedure done, pid=37 2023-10-02T12:34:48,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696250087647_ns2_test-16962500765861 on region 97d4833ae9b729f41ce094e6b5f5d9e0 2023-10-02T12:34:48,707 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 97d4833ae9b729f41ce094e6b5f5d9e0 2023-10-02T12:34:48,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-10-02T12:34:48,711 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:34:48,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 97d4833ae9b729f41ce094e6b5f5d9e0 in 1.0200 sec 2023-10-02T12:34:48,711 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:34:48,712 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:34:48,713 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696250087647_ns2_test-16962500765861 2023-10-02T12:34:48,714 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_1696250087647_ns2_test-16962500765861 2023-10-02T12:34:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:34:49,090 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:34:49,135 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:34:49,149 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:34:49,150 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_1696250087647_ns2_test-16962500765861 to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_1696250087647_ns2_test-16962500765861 2023-10-02T12:34:49,152 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:34:49,152 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-10-02T12:34:49,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696250087647_ns2_test-16962500765861 table=ns2:test-16962500765861 type=FLUSH ttl=0 } in 1.4900 sec 2023-10-02T12:34:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:34:49,777 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: ns2:test-16962500765861, procId: 36 completed 2023-10-02T12:34:49,777 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696250081997 2023-10-02T12:34:49,777 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:34:49,779 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-10-02T12:34:49,781 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696250081997_test-1696250076586 2023-10-02T12:34:49,781 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696250085506_default_test-1696250076586 to hdfs://localhost:33727/backupUT/backup_1696250081997/default/test-1696250076586/ 2023-10-02T12:34:49,791 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:34:49,834 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:33727, inputRoot=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3 2023-10-02T12:34:49,834 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,39919,1696250062021_936791485_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33727/backupUT/backup_1696250081997/default/test-1696250076586, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33727/backupUT/backup_1696250081997/default/test-1696250076586/.hbase-snapshot/.tmp/snapshot_1696250085506_default_test-1696250076586 2023-10-02T12:34:49,834 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:33727, inputRoot=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3, snapshotDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_1696250085506_default_test-1696250076586. 2023-10-02T12:34:49,851 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_1696250085506_default_test-1696250076586 to hdfs://localhost:33727/backupUT/backup_1696250081997/default/test-1696250076586/.hbase-snapshot/.tmp/snapshot_1696250085506_default_test-1696250076586 2023-10-02T12:34:50,293 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:50,294 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:50,294 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:50,295 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:51,421 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop-1379041572795292819.jar 2023-10-02T12:34:51,422 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:51,423 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:51,423 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:51,424 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:51,424 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:51,425 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:51,425 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:51,426 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:51,426 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:51,428 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:34:51,429 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:34:51,429 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:34:51,431 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:34:51,432 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:34:51,433 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:34:51,433 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:34:51,434 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:34:51,435 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:34:51,435 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:34:51,436 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:34:51,436 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:34:51,441 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:34:51,441 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:34:51,442 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:34:51,442 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:34:51,443 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:34:51,443 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:34:51,444 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:34:51,534 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:34:51,571 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696250085506_default_test-1696250076586' hfile list 2023-10-02T12:34:51,582 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:34:51,806 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250091757/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:34:51,806 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250091757/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:34:51,976 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:34:52,054 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/f/a7719d605238452e97f183af22877b79 output=hdfs://localhost:33727/backupUT/backup_1696250081997/default/test-1696250076586/archive/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/f/a7719d605238452e97f183af22877b79 2023-10-02T12:34:52,055 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 2.669M/sec 2023-10-02T12:34:52,867 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:34:52,869 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:34:52,881 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696250085506_default_test-1696250076586 2023-10-02T12:34:52,881 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696250085506_default_test-1696250076586 finished. 2023-10-02T12:34:52,881 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696250081997_ns2:test-16962500765861 2023-10-02T12:34:52,881 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696250087647_ns2_test-16962500765861 to hdfs://localhost:33727/backupUT/backup_1696250081997/ns2/test-16962500765861/ 2023-10-02T12:34:52,881 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:34:52,909 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:33727, inputRoot=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3 2023-10-02T12:34:52,909 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,39919,1696250062021_-1925916422_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33727/backupUT/backup_1696250081997/ns2/test-16962500765861, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33727/backupUT/backup_1696250081997/ns2/test-16962500765861/.hbase-snapshot/.tmp/snapshot_1696250087647_ns2_test-16962500765861 2023-10-02T12:34:52,909 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:33727, inputRoot=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3, snapshotDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_1696250087647_ns2_test-16962500765861. 2023-10-02T12:34:52,929 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_1696250087647_ns2_test-16962500765861 to hdfs://localhost:33727/backupUT/backup_1696250081997/ns2/test-16962500765861/.hbase-snapshot/.tmp/snapshot_1696250087647_ns2_test-16962500765861 2023-10-02T12:34:53,003 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:53,004 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:53,004 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:53,005 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:54,223 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop-3334723762319914919.jar 2023-10-02T12:34:54,223 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:54,224 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:54,225 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:54,225 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:54,226 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:54,227 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:54,227 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:54,227 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:54,228 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:54,229 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:34:54,229 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:34:54,230 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:34:54,230 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:34:54,231 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:34:54,232 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:34:54,232 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:34:54,233 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:34:54,233 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:34:54,234 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:34:54,235 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:34:54,235 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:34:54,236 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:34:54,237 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:34:54,238 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:34:54,238 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:34:54,238 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:34:54,239 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:34:54,240 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:34:54,280 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:34:54,326 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696250087647_ns2_test-16962500765861' hfile list 2023-10-02T12:34:54,330 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:34:54,448 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250094394/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:34:54,448 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250094394/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:34:54,516 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:34:54,542 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/f/4b0589f8b30b4e1cb1ee0b3f2d8f21d5 output=hdfs://localhost:33727/backupUT/backup_1696250081997/ns2/test-16962500765861/archive/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/f/4b0589f8b30b4e1cb1ee0b3f2d8f21d5 2023-10-02T12:34:54,542 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-10-02T12:34:55,494 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:34:55,495 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:34:55,503 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696250087647_ns2_test-16962500765861 2023-10-02T12:34:55,503 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696250087647_ns2_test-16962500765861 finished. 2023-10-02T12:34:55,503 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1034): test-1696250076586 2023-10-02T12:34:55,503 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1034): ns2:test-16962500765861 2023-10-02T12:34:55,589 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250081997 2023-10-02T12:34:55,589 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:34:56,002 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250081997/.backup.manifest 2023-10-02T12:34:56,002 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250081997 2023-10-02T12:34:56,002 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:34:56,409 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250081997/.backup.manifest 2023-10-02T12:34:56,410 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(358): Backup backup_1696250081997 finished: type=FULL,tablelist=test-1696250076586;ns2:test-16962500765861,targetRootDir=hdfs://localhost:33727/backupUT,startts=1696250085191,completets=1696250095586,bytescopied=0 2023-10-02T12:34:56,410 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:34:56,410 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696250085506_default_test-1696250076586 2023-10-02T12:34:56,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696250085506_default_test-1696250076586" 2023-10-02T12:34:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696250085506_default_test-1696250076586 2023-10-02T12:34:56,427 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696250085506_default_test-1696250076586 for backup backup_1696250081997 succeeded. 2023-10-02T12:34:56,427 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696250087647_ns2_test-16962500765861 2023-10-02T12:34:56,428 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696250087647_ns2_test-16962500765861" 2023-10-02T12:34:56,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696250087647_ns2_test-16962500765861 2023-10-02T12:34:56,434 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696250087647_ns2_test-16962500765861 for backup backup_1696250081997 succeeded. 2023-10-02T12:34:56,435 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:34:56,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:34:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:34:56,458 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:34:56,466 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:34:56,488 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(376): Backup backup_1696250081997 completed. 2023-10-02T12:34:56,716 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:34:56,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:34:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250096722 (current time:1696250096722). 2023-10-02T12:34:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:34:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:34:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0a31f8f2 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:34:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd23165, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:56,734 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50094, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0a31f8f2 to 127.0.0.1:64600 2023-10-02T12:34:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:34:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:34:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 38 2023-10-02T12:34:56,741 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:34:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:34:56,743 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:34:56,747 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot, current working dir path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:34:56,750 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:34:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:34:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:34:57,159 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:34:57,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2}] 2023-10-02T12:34:57,162 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:57,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40211 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=39 2023-10-02T12:34:57,315 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:34:57,315 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(2819): Flushing 5bf5964d15ce023a31881482c13240a2 2/2 column families, dataSize=1.51 KB heapSize=2.80 KB 2023-10-02T12:34:57,331 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/534be26151d942a196491c2a2fc799b3 is 171, key is trslm:hdfs://localhost:33727/backupUT\x00ns2:test-16962500765861/meta:log-roll-map/1696250095575/Put/seqid=0 2023-10-02T12:34:57,337 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/534be26151d942a196491c2a2fc799b3 2023-10-02T12:34:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:34:57,349 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/ff8629cab718428281c86435a4858f46 is 459, key is session:backup_1696250081997/session:context/1696250096465/Put/seqid=0 2023-10-02T12:34:57,372 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=758 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/ff8629cab718428281c86435a4858f46 2023-10-02T12:34:57,382 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/534be26151d942a196491c2a2fc799b3 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3 2023-10-02T12:34:57,390 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3, entries=6, sequenceid=17, filesize=5.8 K 2023-10-02T12:34:57,391 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/ff8629cab718428281c86435a4858f46 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46 2023-10-02T12:34:57,399 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46, entries=2, sequenceid=17, filesize=5.4 K 2023-10-02T12:34:57,400 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(3022): Finished flush of dataSize ~1.51 KB/1543, heapSize ~2.77 KB/2840, currentSize=0 B/0 for 5bf5964d15ce023a31881482c13240a2 in 85ms, sequenceid=17, compaction requested=false 2023-10-02T12:34:57,400 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(2520): Flush status journal for 5bf5964d15ce023a31881482c13240a2: 2023-10-02T12:34:57,400 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. for snapshot_backup_system completed. 2023-10-02T12:34:57,401 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:34:57,401 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:34:57,401 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3] hfiles 2023-10-02T12:34:57,401 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3 for snapshot=snapshot_backup_system 2023-10-02T12:34:57,401 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46] hfiles 2023-10-02T12:34:57,401 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10 for snapshot=snapshot_backup_system 2023-10-02T12:34:57,401 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46 for snapshot=snapshot_backup_system 2023-10-02T12:34:57,407 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:34:57,407 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=39 2023-10-02T12:34:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster(3995): Remote procedure done, pid=39 2023-10-02T12:34:57,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:57,409 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:34:57,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=39, resume processing ppid=38 2023-10-02T12:34:57,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, ppid=38, state=SUCCESS; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 in 250 msec 2023-10-02T12:34:57,413 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:34:57,414 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:34:57,415 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:34:57,415 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:34:57,416 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:34:57,829 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:34:57,843 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:34:57,844 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:34:57,846 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:34:57,847 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 38 2023-10-02T12:34:57,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.1090 sec 2023-10-02T12:34:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:34:57,849 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 38 completed 2023-10-02T12:34:57,849 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(115): Backup backup_1696250096681 started at 1696250097849. 2023-10-02T12:34:57,851 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(119): Backup session backup_1696250096681 has been started. 2023-10-02T12:34:57,858 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [ns2:test-16962500765861, test-1696250076586] 2023-10-02T12:34:57,863 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(76): StartCode 1696250068185for backupID backup_1696250096681 2023-10-02T12:34:57,863 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-10-02T12:34:57,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:34:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:34:57,864 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:34:57,864 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:34:57,865 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:34:57,865 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:34:57,866 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:34:57,866 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:34:57,867 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:34:57,868 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,868 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:34:57,868 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:34:57,868 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,868 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:34:57,868 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:34:57,869 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:34:57,869 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:34:57,869 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:34:57,869 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:34:57,870 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:34:57,870 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:34:57,870 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:34:57,870 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:34:57,871 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:34:57,871 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:34:57,871 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:34:57,876 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,876 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:57,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:34:57,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:34:57,877 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:34:57,877 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:34:57,877 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:34:57,877 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:34:57,877 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:57,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:34:57,879 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:34:57,879 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@e0158ee[Count = 0] remaining members to acquire global barrier 2023-10-02T12:34:57,879 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:34:57,879 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:57,880 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:57,881 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:57,881 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:57,881 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:34:57,881 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,881 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:34:57,881 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:34:57,881 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,881 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250085275 highest: 1696250085275 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,881 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770:(num 1696250085275) roll requested 2023-10-02T12:34:57,898 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250097882, exclude list is [], retry=0 2023-10-02T12:34:57,904 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:34:57,907 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250085275 with entries=113, filesize=13.97 KB; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250097882 2023-10-02T12:34:57,907 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:34:57,907 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250085275 is not closed yet, will try archiving it next time 2023-10-02T12:34:57,907 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta:.meta(num 1696250085307) roll requested 2023-10-02T12:34:57,926 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250097907.meta, exclude list is [], retry=0 2023-10-02T12:34:57,931 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:34:57,933 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250085307.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250097907.meta 2023-10-02T12:34:57,933 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:34:57,934 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250085307.meta is not closed yet, will try archiving it next time 2023-10-02T12:34:57,938 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250085307.meta to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250085307.meta 2023-10-02T12:34:57,942 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-10-02T12:34:57,942 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250097882 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,971 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:34:57,971 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40211,1696250065770' in zk 2023-10-02T12:34:57,976 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,976 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:34:57,976 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:34:57,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,976 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:34:57,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:34:57,978 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:34:57,979 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:34:57,980 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:34:57,980 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:57,981 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,981 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:34:57,982 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:57,982 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,983 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40211,1696250065770': 2023-10-02T12:34:57,983 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:34:57,984 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:34:57,984 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:34:57,984 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:34:57,984 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:34:57,984 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:34:57,986 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:34:57,986 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:34:57,986 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:34:57,986 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:34:57,986 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:34:57,986 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:34:57,986 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:34:57,986 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:34:57,987 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:34:57,987 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,987 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:34:57,987 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:34:57,988 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:57,988 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:34:57,989 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:34:57,989 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:57,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,990 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:34:57,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:34:57,992 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,999 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,999 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:34:57,999 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:34:57,999 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:34:57,999 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:34:57,999 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:34:57,999 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:34:57,999 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:57,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:34:58,000 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:34:58,001 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:34:58,001 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:34:58,000 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:34:58,001 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-10-02T12:34:58,001 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:34:58,001 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:34:58,002 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:34:58,002 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:34:58,101 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:34:58,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:34:58,106 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:40211=1696250068185} newestTimestamps: {jenkins-hbase3.apache.org:40211=1696250085275} 2023-10-02T12:34:58,109 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250069157.meta 2023-10-02T12:34:58,109 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250097907.meta 2023-10-02T12:34:58,109 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250085275 2023-10-02T12:34:58,110 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250097882 2023-10-02T12:34:58,110 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250085307.meta 2023-10-02T12:34:58,130 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(146): Attempting to copy table info for:test-1696250076586 target: hdfs://localhost:33727/backupUT/backup_1696250096681/default/test-1696250076586 descriptor: 'test-1696250076586', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:34:58,131 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-10-02T12:34:58,134 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(151): Starting to write region info for table test-1696250076586 2023-10-02T12:34:58,155 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(158): Finished writing region info for table test-1696250076586 2023-10-02T12:34:58,174 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16962500765861 target: hdfs://localhost:33727/backupUT/backup_1696250096681/ns2/test-16962500765861 descriptor: 'ns2:test-16962500765861', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:34:58,174 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-10-02T12:34:58,177 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16962500765861 2023-10-02T12:34:58,185 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16962500765861 2023-10-02T12:34:58,195 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:33727/backupUT/.tmp/backup_1696250096681 from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250085275 2023-10-02T12:34:58,200 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x516f7a3e to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:34:58,208 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db620b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:34:58,210 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:34:58,213 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45704, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:34:58,234 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:34:58,234 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16962500765861 2023-10-02T12:34:58,242 DEBUG [hconnection-0x44440b72-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:34:58,243 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50110, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:34:58,248 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16962500765861: ns2:test-16962500765861; 2023-10-02T12:34:58,249 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1696250076586 2023-10-02T12:34:58,252 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1696250076586: test-1696250076586; 2023-10-02T12:34:58,252 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-10-02T12:34:58,252 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/hbase-staging/partitions_9409624e-aa7d-4ff7-b14b-d6645d77242c 2023-10-02T12:34:58,315 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:58,316 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:58,317 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:58,319 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,717 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop-8318642831558204881.jar 2023-10-02T12:34:59,718 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,718 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,719 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,720 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,721 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,721 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,722 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,722 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,723 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,724 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:34:59,725 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:34:59,725 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:34:59,726 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:34:59,727 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:34:59,727 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:34:59,728 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:34:59,728 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:34:59,729 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:34:59,729 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:34:59,730 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:34:59,731 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:34:59,734 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,734 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,735 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,735 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,736 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,736 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:34:59,737 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:34:59,738 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16962500765861,default:test-1696250076586 2023-10-02T12:34:59,738 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:34:59,739 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x516f7a3e to 127.0.0.1:64600 2023-10-02T12:34:59,739 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:34:59,740 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:35:01,061 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop-5904171901944514981.jar 2023-10-02T12:35:01,069 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:35:01,106 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250085275 for WAL files 2023-10-02T12:35:01,108 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250085275 2023-10-02T12:35:01,222 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250101179/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:35:01,222 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250101179/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:35:01,348 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250085275 for hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250085275 (-9223372036854775808:9223372036854775807) length:14316 2023-10-02T12:35:01,367 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-10-02T12:35:01,368 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-10-02T12:35:01,560 DEBUG [pool-204-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4ec21078 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:01,567 DEBUG [pool-204-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@523195d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:01,574 DEBUG [hconnection-0x5c3a20cf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:01,577 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60490, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:01,580 DEBUG [pool-204-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4ec21078 to 127.0.0.1:64600 2023-10-02T12:35:01,580 DEBUG [pool-204-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:01,582 DEBUG [pool-204-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t10] 2023-10-02T12:35:01,582 DEBUG [pool-204-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-10-02T12:35:01,597 DEBUG [pool-204-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/backupUT/.tmp/backup_1696250096681/_temporary/0/_temporary/attempt_local162591407_0003_r_000001_0/default/test-1696250076586/f/eb6e2b1f43f14c168646c93971249f00 is 40, key is row-t110/f:q1/1696250096528/Put/seqid=0 2023-10-02T12:35:01,601 WARN [Thread-847 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:40211] Specified favored nodes: [jenkins-hbase3.apache.org:40211] 2023-10-02T12:35:01,607 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:35:01,609 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49480, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:35:02,266 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:33727/backupUT 2023-10-02T12:35:02,267 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1696250096681 2023-10-02T12:35:02,267 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-10-02T12:35:02,284 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:33727/backupUT/.tmp/backup_1696250096681, hdfs://localhost:33727/backupUT] 2023-10-02T12:35:02,355 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:35:02,956 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-10-02T12:35:02,959 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1696250096681 2023-10-02T12:35:02,959 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 8796 bytes copied." 2023-10-02T12:35:02,960 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local224417992_0004 completed: true true 2023-10-02T12:35:02,969 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=3775865 FILE: Number of bytes written=5055358 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=107127 HDFS: Number of bytes written=2323069 HDFS: Number of read operations=443 HDFS: Number of large read operations=0 HDFS: Number of write operations=188 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=343 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1344798720 File Input Format Counters Bytes Read=1478 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=8796 Bytes Expected=8796 Files Copied=2 DIR_COPY=4 2023-10-02T12:35:02,969 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:33727/backupUT/.tmp/backup_1696250096681 to hdfs://localhost:33727/backupUT finished. 2023-10-02T12:35:02,992 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-10-02T12:35:02,992 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250096681 2023-10-02T12:35:03,002 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-10-02T12:35:03,002 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250096681 2023-10-02T12:35:03,004 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-10-02T12:35:03,013 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:03,013 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250096681 2023-10-02T12:35:03,015 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-10-02T12:35:03,015 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250096681 2023-10-02T12:35:03,017 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-10-02T12:35:03,023 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:03,023 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250096681 2023-10-02T12:35:03,025 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-10-02T12:35:03,432 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:03,432 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(358): Backup backup_1696250096681 finished: type=INCREMENTAL,tablelist=test-1696250076586;ns2:test-16962500765861,targetRootDir=hdfs://localhost:33727/backupUT,startts=1696250097849,completets=1696250102992,bytescopied=0 2023-10-02T12:35:03,432 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:35:03,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:35:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:35:03,442 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:35:03,444 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:35:03,448 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(376): Backup backup_1696250096681 completed. 2023-10-02T12:35:03,643 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:35:03,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:35:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250103648 (current time:1696250103648). 2023-10-02T12:35:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:35:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:35:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7ec8b204 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d3bea9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:03,661 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60502, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7ec8b204 to 127.0.0.1:64600 2023-10-02T12:35:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:35:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=40, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:35:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 40 2023-10-02T12:35:03,665 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:35:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-10-02T12:35:03,667 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:35:03,670 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot, current working dir path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:35:03,673 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:35:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-10-02T12:35:03,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-10-02T12:35:04,097 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:35:04,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2}] 2023-10-02T12:35:04,099 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:04,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40211 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=41 2023-10-02T12:35:04,252 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:04,252 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(2819): Flushing 5bf5964d15ce023a31881482c13240a2 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-10-02T12:35:04,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/6902999acb1740f2bc2bc521b427574b is 171, key is trslm:hdfs://localhost:33727/backupUT\x00ns2:test-16962500765861/meta:log-roll-map/1696250102975/Put/seqid=0 2023-10-02T12:35:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-10-02T12:35:04,663 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/6902999acb1740f2bc2bc521b427574b 2023-10-02T12:35:04,677 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/ea7b382616974833926d102b744ceab7 is 360, key is session:backup_1696250096681/session:context/1696250103443/Put/seqid=0 2023-10-02T12:35:04,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-10-02T12:35:05,084 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/ea7b382616974833926d102b744ceab7 2023-10-02T12:35:05,095 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/6902999acb1740f2bc2bc521b427574b as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b 2023-10-02T12:35:05,104 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b, entries=4, sequenceid=28, filesize=5.7 K 2023-10-02T12:35:05,106 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/ea7b382616974833926d102b744ceab7 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7 2023-10-02T12:35:05,115 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7, entries=2, sequenceid=28, filesize=5.3 K 2023-10-02T12:35:05,116 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for 5bf5964d15ce023a31881482c13240a2 in 864ms, sequenceid=28, compaction requested=true 2023-10-02T12:35:05,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(2520): Flush status journal for 5bf5964d15ce023a31881482c13240a2: 2023-10-02T12:35:05,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. for snapshot_backup_system completed. 2023-10-02T12:35:05,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:35:05,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:35:05,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b] hfiles 2023-10-02T12:35:05,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3 for snapshot=snapshot_backup_system 2023-10-02T12:35:05,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b for snapshot=snapshot_backup_system 2023-10-02T12:35:05,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7] hfiles 2023-10-02T12:35:05,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10 for snapshot=snapshot_backup_system 2023-10-02T12:35:05,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46 for snapshot=snapshot_backup_system 2023-10-02T12:35:05,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7 for snapshot=snapshot_backup_system 2023-10-02T12:35:05,125 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:05,125 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=41 2023-10-02T12:35:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster(3995): Remote procedure done, pid=41 2023-10-02T12:35:05,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:05,127 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:05,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-10-02T12:35:05,130 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:35:05,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 in 1.0310 sec 2023-10-02T12:35:05,131 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:35:05,131 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:35:05,131 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:35:05,132 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:35:05,547 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:35:05,559 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:35:05,560 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:35:05,561 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:35:05,561 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 40 2023-10-02T12:35:05,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8980 sec 2023-10-02T12:35:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=40 2023-10-02T12:35:05,774 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 40 completed 2023-10-02T12:35:05,774 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(115): Backup backup_1696250103611 started at 1696250105774. 2023-10-02T12:35:05,776 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(119): Backup session backup_1696250103611 has been started. 2023-10-02T12:35:05,778 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [ns2:test-16962500765861, test-1696250076586] 2023-10-02T12:35:05,784 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(76): StartCode 1696250085275for backupID backup_1696250103611 2023-10-02T12:35:05,784 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-10-02T12:35:05,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:35:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:35:05,786 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:35:05,786 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:35:05,787 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:35:05,787 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:35:05,787 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:05,787 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:05,789 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:35:05,789 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,789 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:35:05,789 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:35:05,789 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,789 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:35:05,790 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:05,790 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:05,790 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:35:05,790 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:05,790 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:35:05,791 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:35:05,791 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:35:05,791 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:35:05,791 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:35:05,792 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:35:05,792 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:35:05,792 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:35:05,799 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:05,799 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,800 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,800 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:05,800 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:05,800 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:35:05,800 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:35:05,800 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:05,800 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:05,801 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:05,801 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,801 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:05,802 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:35:05,802 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@48657cb6[Count = 0] remaining members to acquire global barrier 2023-10-02T12:35:05,802 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:35:05,802 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:05,803 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:05,803 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:05,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:05,803 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:35:05,803 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,803 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:35:05,803 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:35:05,804 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,804 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250097882 highest: 1696250097882 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,804 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770:(num 1696250097882) roll requested 2023-10-02T12:35:05,822 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250105804, exclude list is [], retry=0 2023-10-02T12:35:05,826 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:35:05,829 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250097882 with entries=109, filesize=13.33 KB; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250105804 2023-10-02T12:35:05,829 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:35:05,829 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250097882 is not closed yet, will try archiving it next time 2023-10-02T12:35:05,829 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta:.meta(num 1696250097907) roll requested 2023-10-02T12:35:05,842 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250105829.meta, exclude list is [], retry=0 2023-10-02T12:35:05,846 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:35:05,848 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250097907.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250105829.meta 2023-10-02T12:35:05,848 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:35:05,849 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250097907.meta is not closed yet, will try archiving it next time 2023-10-02T12:35:05,851 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250097907.meta not finished, retry = 0 2023-10-02T12:35:05,864 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-10-02T12:35:05,864 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250105804 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,875 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:35:05,875 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40211,1696250065770' in zk 2023-10-02T12:35:05,877 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:35:05,877 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,877 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:35:05,877 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:35:05,879 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,879 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:05,879 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:05,879 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:05,879 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:05,880 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:05,880 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,881 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:05,881 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:05,881 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,882 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40211,1696250065770': 2023-10-02T12:35:05,882 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:35:05,882 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:35:05,882 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:35:05,882 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:35:05,882 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:35:05,882 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:35:05,884 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:05,884 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:05,884 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:35:05,885 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:05,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:05,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:05,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:05,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:05,885 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:35:05,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:35:05,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:05,885 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:05,886 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:05,886 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:05,887 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:05,887 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,887 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,887 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:05,888 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:05,888 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,891 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,891 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:05,891 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:05,891 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:35:05,891 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:05,891 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:05,891 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:35:05,891 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:05,891 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:35:05,891 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:05,891 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:35:05,891 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:35:05,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:35:05,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:35:05,893 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:35:05,893 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:35:05,893 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-10-02T12:35:05,893 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:35:05,953 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250097907.meta to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250097907.meta 2023-10-02T12:35:05,993 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:35:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:35:05,997 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:40211=1696250085275} newestTimestamps: {jenkins-hbase3.apache.org:40211=1696250097882} 2023-10-02T12:35:05,999 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250069157.meta 2023-10-02T12:35:05,999 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250105829.meta 2023-10-02T12:35:05,999 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250097882 2023-10-02T12:35:05,999 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250105804 2023-10-02T12:35:05,999 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250085307.meta 2023-10-02T12:35:05,999 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250097907.meta 2023-10-02T12:35:06,411 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(146): Attempting to copy table info for:test-1696250076586 target: hdfs://localhost:33727/backupUT/backup_1696250103611/default/test-1696250076586 descriptor: 'test-1696250076586', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:35:06,411 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-10-02T12:35:06,413 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(151): Starting to write region info for table test-1696250076586 2023-10-02T12:35:06,819 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(158): Finished writing region info for table test-1696250076586 2023-10-02T12:35:07,232 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16962500765861 target: hdfs://localhost:33727/backupUT/backup_1696250103611/ns2/test-16962500765861 descriptor: 'ns2:test-16962500765861', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:35:07,232 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-10-02T12:35:07,234 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16962500765861 2023-10-02T12:35:07,240 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16962500765861 2023-10-02T12:35:07,245 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:33727/backupUT/.tmp/backup_1696250103611 from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250097882 2023-10-02T12:35:07,246 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x59c7b230 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:07,252 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b51fa72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:07,253 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:35:07,255 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49488, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:35:07,257 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:35:07,257 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16962500765861 2023-10-02T12:35:07,262 DEBUG [hconnection-0x741a9d9d-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:07,263 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60504, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:07,264 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16962500765861: ns2:test-16962500765861; 2023-10-02T12:35:07,264 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1696250076586 2023-10-02T12:35:07,266 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1696250076586: test-1696250076586; 2023-10-02T12:35:07,266 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-10-02T12:35:07,267 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/hbase-staging/partitions_e721bb5f-83a6-4732-95fc-1850400f8916 2023-10-02T12:35:07,674 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:07,675 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:07,676 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:07,676 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,712 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop-7548030388488236522.jar 2023-10-02T12:35:08,713 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,713 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,714 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,714 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,715 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,715 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,715 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,716 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,716 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,717 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:35:08,718 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:35:08,718 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:35:08,718 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:35:08,719 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:35:08,719 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:35:08,720 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:35:08,720 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:35:08,720 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:35:08,721 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:35:08,721 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:35:08,722 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:35:08,722 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,723 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,723 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,724 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,724 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,725 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:08,725 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:35:08,726 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16962500765861,default:test-1696250076586 2023-10-02T12:35:08,726 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:35:08,726 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x59c7b230 to 127.0.0.1:64600 2023-10-02T12:35:08,726 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:08,727 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:35:09,724 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop-4419720402398124447.jar 2023-10-02T12:35:09,729 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:35:09,748 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250097882 for WAL files 2023-10-02T12:35:09,750 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250097882 2023-10-02T12:35:09,851 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250109807/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:35:09,851 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250109807/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:35:09,914 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250097882 for hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250097882 (-9223372036854775808:9223372036854775807) length:13653 2023-10-02T12:35:09,924 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-10-02T12:35:09,924 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-10-02T12:35:09,942 DEBUG [pool-217-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x20a17970 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:09,947 DEBUG [pool-217-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@715512a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:09,949 DEBUG [hconnection-0x427ad21-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:09,951 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60516, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:09,953 DEBUG [pool-217-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x20a17970 to 127.0.0.1:64600 2023-10-02T12:35:09,953 DEBUG [pool-217-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:09,955 DEBUG [pool-217-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-10-02T12:35:09,955 DEBUG [pool-217-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-10-02T12:35:09,967 DEBUG [pool-217-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/backupUT/.tmp/backup_1696250103611/_temporary/0/_temporary/attempt_local1796736330_0005_r_000000_0/ns2/test-16962500765861/f/020b686cdd184d53aefd0fb96d34a5d6 is 40, key is row-t210/f:q1/1696250103484/Put/seqid=0 2023-10-02T12:35:09,969 WARN [Thread-972 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:40211] Specified favored nodes: [jenkins-hbase3.apache.org:40211] 2023-10-02T12:35:10,654 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-10-02T12:35:10,654 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-10-02T12:35:10,906 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:33727/backupUT 2023-10-02T12:35:10,906 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1696250103611 2023-10-02T12:35:10,906 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-10-02T12:35:10,913 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:33727/backupUT/.tmp/backup_1696250103611, hdfs://localhost:33727/backupUT] 2023-10-02T12:35:10,941 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:35:11,522 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-10-02T12:35:11,530 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1696250103611 2023-10-02T12:35:11,530 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 9114 bytes copied." 2023-10-02T12:35:11,531 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1708830335_0006 completed: true true 2023-10-02T12:35:11,535 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=4412160 FILE: Number of bytes written=7464356 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=153820 HDFS: Number of bytes written=2356042 HDFS: Number of read operations=564 HDFS: Number of large read operations=0 HDFS: Number of write operations=231 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1344798720 File Input Format Counters Bytes Read=1452 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=9114 Bytes Expected=9114 Files Copied=2 DIR_COPY=4 2023-10-02T12:35:11,535 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:33727/backupUT/.tmp/backup_1696250103611 to hdfs://localhost:33727/backupUT finished. 2023-10-02T12:35:11,548 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-10-02T12:35:11,548 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250103611 2023-10-02T12:35:11,554 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:11,554 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:11,558 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:11,558 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:11,558 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-10-02T12:35:11,558 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250103611 2023-10-02T12:35:11,560 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:11,560 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:11,563 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:11,563 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:11,563 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-10-02T12:35:11,572 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:11,572 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250103611 2023-10-02T12:35:11,576 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:11,576 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:11,580 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:11,580 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:11,580 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-10-02T12:35:11,580 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250103611 2023-10-02T12:35:11,583 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:11,583 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:11,586 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:11,586 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:11,586 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-10-02T12:35:11,997 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:11,997 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250103611 2023-10-02T12:35:11,999 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:11,999 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:12,002 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:12,003 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:12,003 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-10-02T12:35:12,009 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:12,009 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(358): Backup backup_1696250103611 finished: type=INCREMENTAL,tablelist=test-1696250076586;ns2:test-16962500765861,targetRootDir=hdfs://localhost:33727/backupUT,startts=1696250105774,completets=1696250111548,bytescopied=0 2023-10-02T12:35:12,009 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:35:12,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:35:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:35:12,019 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:35:12,021 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:35:12,024 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(376): Backup backup_1696250103611 completed. 2023-10-02T12:35:12,180 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:35:12,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:35:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250112184 (current time:1696250112184). 2023-10-02T12:35:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:35:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:35:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1c4a4d81 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b17b83f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:12,196 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41506, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1c4a4d81 to 127.0.0.1:64600 2023-10-02T12:35:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:12,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:35:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:35:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-10-02T12:35:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:35:12,201 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:35:12,202 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:35:12,205 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot, current working dir path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:35:12,207 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:35:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:35:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:35:12,615 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:35:12,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2}] 2023-10-02T12:35:12,618 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:12,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40211 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=43 2023-10-02T12:35:12,770 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:12,770 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2819): Flushing 5bf5964d15ce023a31881482c13240a2 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-10-02T12:35:12,775 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/a17f8975f69648a7b3a6441fe977b883 is 171, key is trslm:hdfs://localhost:33727/backupUT\x00ns2:test-16962500765861/meta:log-roll-map/1696250111540/Put/seqid=0 2023-10-02T12:35:12,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:35:13,188 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/a17f8975f69648a7b3a6441fe977b883 2023-10-02T12:35:13,199 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/59914df0bc484059b34c6b7acf8ae537 is 360, key is session:backup_1696250103611/session:context/1696250112020/Put/seqid=0 2023-10-02T12:35:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:35:13,605 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/59914df0bc484059b34c6b7acf8ae537 2023-10-02T12:35:13,614 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/a17f8975f69648a7b3a6441fe977b883 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/a17f8975f69648a7b3a6441fe977b883 2023-10-02T12:35:13,644 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/a17f8975f69648a7b3a6441fe977b883, entries=4, sequenceid=39, filesize=5.7 K 2023-10-02T12:35:13,646 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/59914df0bc484059b34c6b7acf8ae537 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/59914df0bc484059b34c6b7acf8ae537 2023-10-02T12:35:13,657 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/59914df0bc484059b34c6b7acf8ae537, entries=2, sequenceid=39, filesize=5.3 K 2023-10-02T12:35:13,662 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for 5bf5964d15ce023a31881482c13240a2 in 892ms, sequenceid=39, compaction requested=true 2023-10-02T12:35:13,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2520): Flush status journal for 5bf5964d15ce023a31881482c13240a2: 2023-10-02T12:35:13,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. for snapshot_backup_system completed. 2023-10-02T12:35:13,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:35:13,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:35:13,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/a17f8975f69648a7b3a6441fe977b883] hfiles 2023-10-02T12:35:13,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3 for snapshot=snapshot_backup_system 2023-10-02T12:35:13,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b for snapshot=snapshot_backup_system 2023-10-02T12:35:13,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/a17f8975f69648a7b3a6441fe977b883 for snapshot=snapshot_backup_system 2023-10-02T12:35:13,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/59914df0bc484059b34c6b7acf8ae537] hfiles 2023-10-02T12:35:13,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10 for snapshot=snapshot_backup_system 2023-10-02T12:35:13,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46 for snapshot=snapshot_backup_system 2023-10-02T12:35:13,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7 for snapshot=snapshot_backup_system 2023-10-02T12:35:13,663 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/59914df0bc484059b34c6b7acf8ae537 for snapshot=snapshot_backup_system 2023-10-02T12:35:14,070 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:14,071 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2023-10-02T12:35:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster(3995): Remote procedure done, pid=43 2023-10-02T12:35:14,071 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:14,073 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:14,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-10-02T12:35:14,076 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:35:14,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 in 1.4590 sec 2023-10-02T12:35:14,077 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:35:14,078 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:35:14,078 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:35:14,079 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:35:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:35:14,499 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:35:14,515 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:35:14,516 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:35:14,518 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:35:14,519 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-10-02T12:35:14,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.3210 sec 2023-10-02T12:35:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:35:16,309 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 42 completed 2023-10-02T12:35:16,309 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(115): Backup backup_1696250112152 started at 1696250116309. 2023-10-02T12:35:16,311 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(119): Backup session backup_1696250112152 has been started. 2023-10-02T12:35:16,313 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [ns2:test-16962500765861, test-1696250076586] 2023-10-02T12:35:16,320 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(76): StartCode 1696250097882for backupID backup_1696250112152 2023-10-02T12:35:16,320 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-10-02T12:35:16,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:35:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-10-02T12:35:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:35:16,321 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:35:16,321 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:35:16,322 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:35:16,322 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:35:16,323 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:16,323 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:16,324 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:35:16,324 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,324 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:35:16,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:35:16,325 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,325 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:35:16,325 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:16,325 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:16,326 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:35:16,326 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:16,326 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:35:16,326 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:35:16,326 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-10-02T12:35:16,326 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:35:16,326 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:35:16,326 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:35:16,327 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:35:16,327 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:35:16,327 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:35:16,330 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,330 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:16,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:16,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:16,330 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:35:16,331 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:35:16,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:16,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:16,332 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:16,332 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,332 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:16,333 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:35:16,333 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5cdb53a4[Count = 0] remaining members to acquire global barrier 2023-10-02T12:35:16,333 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:35:16,333 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:16,335 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,335 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:35:16,338 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:16,338 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:16,338 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:16,338 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:35:16,339 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:35:16,339 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,339 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250105804 highest: 1696250105804 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,339 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770:(num 1696250105804) roll requested 2023-10-02T12:35:16,350 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250116339, exclude list is [], retry=0 2023-10-02T12:35:16,355 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:35:16,357 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250105804 with entries=109, filesize=13.07 KB; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250116339 2023-10-02T12:35:16,358 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:35:16,358 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250105804 is not closed yet, will try archiving it next time 2023-10-02T12:35:16,358 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta:.meta(num 1696250105829) roll requested 2023-10-02T12:35:16,377 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250116358.meta, exclude list is [], retry=0 2023-10-02T12:35:16,381 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-10-02T12:35:16,381 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250116339 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,382 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:35:16,387 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250105829.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250116358.meta 2023-10-02T12:35:16,387 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:35:16,387 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250105829.meta is not closed yet, will try archiving it next time 2023-10-02T12:35:16,390 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250105829.meta not finished, retry = 0 2023-10-02T12:35:16,411 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:35:16,411 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40211,1696250065770' in zk 2023-10-02T12:35:16,414 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,414 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,414 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:35:16,414 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:16,414 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:35:16,414 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:16,414 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:35:16,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:16,417 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:16,417 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:16,418 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,418 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:16,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:16,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,420 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40211,1696250065770': 2023-10-02T12:35:16,420 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:35:16,420 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:35:16,420 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:35:16,420 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:35:16,420 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:35:16,420 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:35:16,423 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:16,423 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:35:16,423 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:16,423 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:16,423 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:16,424 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:35:16,424 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:16,424 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:35:16,424 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:16,424 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:16,424 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:16,425 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:16,425 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,426 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:16,427 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:16,429 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:16,430 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,430 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,431 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:16,431 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:16,431 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,434 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,434 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:16,434 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:16,434 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:16,434 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:35:16,434 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:16,434 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:35:16,434 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:16,434 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:35:16,435 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:35:16,435 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:16,435 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:35:16,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:35:16,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:35:16,435 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-10-02T12:35:16,435 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:35:16,435 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:35:16,435 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-10-02T12:35:16,435 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:35:16,491 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250105829.meta to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250105829.meta 2023-10-02T12:35:16,536 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:35:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:35:16,540 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:40211=1696250097882} newestTimestamps: {jenkins-hbase3.apache.org:40211=1696250105804} 2023-10-02T12:35:16,542 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250069157.meta 2023-10-02T12:35:16,542 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250116358.meta 2023-10-02T12:35:16,542 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250105804 2023-10-02T12:35:16,542 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250116339 2023-10-02T12:35:16,543 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250085307.meta 2023-10-02T12:35:16,543 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250097907.meta 2023-10-02T12:35:16,543 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250105829.meta 2023-10-02T12:35:16,566 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(146): Attempting to copy table info for:test-1696250076586 target: hdfs://localhost:33727/backupUT/backup_1696250112152/default/test-1696250076586 descriptor: 'test-1696250076586', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:35:16,566 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-10-02T12:35:16,572 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(151): Starting to write region info for table test-1696250076586 2023-10-02T12:35:16,985 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(158): Finished writing region info for table test-1696250076586 2023-10-02T12:35:17,397 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16962500765861 target: hdfs://localhost:33727/backupUT/backup_1696250112152/ns2/test-16962500765861 descriptor: 'ns2:test-16962500765861', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:35:17,397 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-10-02T12:35:17,399 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16962500765861 2023-10-02T12:35:17,805 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16962500765861 2023-10-02T12:35:17,811 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:33727/backupUT/.tmp/backup_1696250112152 from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250105804 2023-10-02T12:35:17,812 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7b5b608d to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:17,819 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65342d29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:17,820 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:35:17,822 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43070, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:35:17,824 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:35:17,824 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16962500765861 2023-10-02T12:35:17,827 DEBUG [hconnection-0x7ae10f16-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:17,829 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:41520, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:17,830 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16962500765861: ns2:test-16962500765861; 2023-10-02T12:35:17,831 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1696250076586 2023-10-02T12:35:17,832 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1696250076586: test-1696250076586; 2023-10-02T12:35:17,832 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-10-02T12:35:17,833 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/hbase-staging/partitions_9f674cf1-0254-48dd-b60b-789bcb0c42e2 2023-10-02T12:35:18,241 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:18,241 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:18,242 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:18,242 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,245 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop-7692431239854452443.jar 2023-10-02T12:35:19,246 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,246 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,246 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,247 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,247 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,248 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,248 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,248 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,249 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,249 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:35:19,250 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:35:19,250 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:35:19,251 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:35:19,251 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:35:19,252 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:35:19,252 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:35:19,252 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:35:19,253 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:35:19,253 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:35:19,254 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:35:19,254 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:35:19,255 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,255 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,255 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,256 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,256 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,256 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:19,257 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:35:19,257 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16962500765861,default:test-1696250076586 2023-10-02T12:35:19,257 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:35:19,258 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7b5b608d to 127.0.0.1:64600 2023-10-02T12:35:19,258 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:19,259 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:35:20,360 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop-4843919035853624912.jar 2023-10-02T12:35:20,366 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:35:20,384 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250105804 for WAL files 2023-10-02T12:35:20,385 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250105804 2023-10-02T12:35:20,464 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250120422/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:35:20,464 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250120422/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:35:20,537 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250105804 for hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250105804 (-9223372036854775808:9223372036854775807) length:13396 2023-10-02T12:35:20,545 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-10-02T12:35:20,545 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-10-02T12:35:20,597 DEBUG [pool-230-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2fc98f0c to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:20,605 DEBUG [pool-230-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7652ea51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:20,609 DEBUG [hconnection-0x49906d1b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:20,612 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50916, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:20,626 DEBUG [pool-230-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2fc98f0c to 127.0.0.1:64600 2023-10-02T12:35:20,626 DEBUG [pool-230-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:20,629 DEBUG [pool-230-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t1100] 2023-10-02T12:35:20,629 DEBUG [pool-230-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-10-02T12:35:20,644 DEBUG [pool-230-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/backupUT/.tmp/backup_1696250112152/_temporary/0/_temporary/attempt_local2043347847_0007_r_000001_0/default/test-1696250076586/f/7b8a335bd3ee4ef69ef14b324cfb38a8 is 42, key is row-t1100/f:q1/1696250112041/Put/seqid=0 2023-10-02T12:35:20,648 WARN [Thread-1107 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:40211] Specified favored nodes: [jenkins-hbase3.apache.org:40211] 2023-10-02T12:35:21,515 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:33727/backupUT 2023-10-02T12:35:21,515 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1696250112152 2023-10-02T12:35:21,515 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-10-02T12:35:21,521 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:33727/backupUT/.tmp/backup_1696250112152, hdfs://localhost:33727/backupUT] 2023-10-02T12:35:21,550 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:35:22,155 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-10-02T12:35:22,157 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1696250112152 2023-10-02T12:35:22,157 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 9086 bytes copied." 2023-10-02T12:35:22,157 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1335000053_0008 completed: true true 2023-10-02T12:35:22,160 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=5052352 FILE: Number of bytes written=9873083 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=202337 HDFS: Number of bytes written=2391182 HDFS: Number of read operations=697 HDFS: Number of large read operations=0 HDFS: Number of write operations=274 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1330642944 File Input Format Counters Bytes Read=1478 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=9086 Bytes Expected=9086 Files Copied=2 DIR_COPY=4 2023-10-02T12:35:22,160 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:33727/backupUT/.tmp/backup_1696250112152 to hdfs://localhost:33727/backupUT finished. 2023-10-02T12:35:22,171 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-10-02T12:35:22,171 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250112152 2023-10-02T12:35:22,175 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:22,175 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:22,178 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:22,178 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:22,178 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:22,178 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:22,181 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:22,181 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:22,181 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-10-02T12:35:22,181 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250112152 2023-10-02T12:35:22,183 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:22,183 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:22,188 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:22,188 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:22,188 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:22,188 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:22,190 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:22,190 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:22,190 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-10-02T12:35:22,596 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:22,596 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250112152 2023-10-02T12:35:22,598 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:22,598 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:22,601 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:22,601 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:22,601 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:22,601 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:22,604 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:22,604 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:22,604 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-10-02T12:35:22,604 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250112152 2023-10-02T12:35:22,606 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:22,606 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:22,608 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:22,609 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:22,609 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:22,609 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:22,611 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:22,611 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:22,611 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-10-02T12:35:23,017 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:23,017 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250112152 2023-10-02T12:35:23,020 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:23,020 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:23,023 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:23,023 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:23,023 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:23,023 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:23,026 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:23,026 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:23,026 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-10-02T12:35:23,431 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:23,432 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(358): Backup backup_1696250112152 finished: type=INCREMENTAL,tablelist=test-1696250076586;ns2:test-16962500765861,targetRootDir=hdfs://localhost:33727/backupUT,startts=1696250116309,completets=1696250122171,bytescopied=0 2023-10-02T12:35:23,432 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:35:23,437 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:35:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:35:23,440 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:35:23,442 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:35:23,445 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(376): Backup backup_1696250112152 completed. 2023-10-02T12:35:23,597 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:35:23,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:35:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250123602 (current time:1696250123602). 2023-10-02T12:35:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:35:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:35:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x38ee6b90 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b4229b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:23,613 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50932, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x38ee6b90 to 127.0.0.1:64600 2023-10-02T12:35:23,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:35:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=44, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:35:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 44 2023-10-02T12:35:23,619 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:35:23,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:35:23,620 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:35:23,623 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot, current working dir path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:35:23,625 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:35:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:35:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:35:24,031 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:35:24,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2}] 2023-10-02T12:35:24,033 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:24,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40211 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=45 2023-10-02T12:35:24,185 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:24,186 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(2819): Flushing 5bf5964d15ce023a31881482c13240a2 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-10-02T12:35:24,190 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/868b0b987e454291983936f0091fe2e3 is 171, key is trslm:hdfs://localhost:33727/backupUT\x00ns2:test-16962500765861/meta:log-roll-map/1696250122165/Put/seqid=0 2023-10-02T12:35:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:35:24,594 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/868b0b987e454291983936f0091fe2e3 2023-10-02T12:35:24,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/ffabe0f68c9244c6936357c580309e5b is 360, key is session:backup_1696250112152/session:context/1696250123441/Put/seqid=0 2023-10-02T12:35:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:35:25,008 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/ffabe0f68c9244c6936357c580309e5b 2023-10-02T12:35:25,015 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/868b0b987e454291983936f0091fe2e3 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/868b0b987e454291983936f0091fe2e3 2023-10-02T12:35:25,021 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/868b0b987e454291983936f0091fe2e3, entries=4, sequenceid=50, filesize=5.7 K 2023-10-02T12:35:25,023 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/ffabe0f68c9244c6936357c580309e5b as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ffabe0f68c9244c6936357c580309e5b 2023-10-02T12:35:25,029 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ffabe0f68c9244c6936357c580309e5b, entries=2, sequenceid=50, filesize=5.3 K 2023-10-02T12:35:25,030 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for 5bf5964d15ce023a31881482c13240a2 in 845ms, sequenceid=50, compaction requested=true 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(2520): Flush status journal for 5bf5964d15ce023a31881482c13240a2: 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. for snapshot_backup_system completed. 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/a17f8975f69648a7b3a6441fe977b883, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/868b0b987e454291983936f0091fe2e3] hfiles 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3 for snapshot=snapshot_backup_system 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b for snapshot=snapshot_backup_system 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/a17f8975f69648a7b3a6441fe977b883 for snapshot=snapshot_backup_system 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/868b0b987e454291983936f0091fe2e3 for snapshot=snapshot_backup_system 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/59914df0bc484059b34c6b7acf8ae537, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ffabe0f68c9244c6936357c580309e5b] hfiles 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10 for snapshot=snapshot_backup_system 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46 for snapshot=snapshot_backup_system 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7 for snapshot=snapshot_backup_system 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/59914df0bc484059b34c6b7acf8ae537 for snapshot=snapshot_backup_system 2023-10-02T12:35:25,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ffabe0f68c9244c6936357c580309e5b for snapshot=snapshot_backup_system 2023-10-02T12:35:25,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:25,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=45 2023-10-02T12:35:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster(3995): Remote procedure done, pid=45 2023-10-02T12:35:25,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:25,440 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:25,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-10-02T12:35:25,442 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:35:25,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 in 1.4100 sec 2023-10-02T12:35:25,443 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:35:25,444 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:35:25,444 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:35:25,445 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:35:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:35:25,858 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:35:25,870 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:35:25,870 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:35:25,871 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:35:25,871 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 44 2023-10-02T12:35:25,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2560 sec 2023-10-02T12:35:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:35:27,729 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 44 completed 2023-10-02T12:35:27,729 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(115): Backup backup_1696250123563 started at 1696250127729. 2023-10-02T12:35:27,731 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(119): Backup session backup_1696250123563 has been started. 2023-10-02T12:35:27,732 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [ns2:test-16962500765861, test-1696250076586] 2023-10-02T12:35:27,738 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(76): StartCode 1696250105804for backupID backup_1696250123563 2023-10-02T12:35:27,738 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-10-02T12:35:27,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:35:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:35:27,739 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:35:27,739 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:35:27,740 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:35:27,740 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:35:27,741 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:27,741 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:27,742 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,742 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:35:27,742 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:35:27,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:35:27,743 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,743 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:35:27,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:27,743 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:27,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:35:27,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:27,744 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:35:27,744 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:35:27,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:35:27,744 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:35:27,744 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:35:27,745 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:35:27,745 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:35:27,745 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:35:27,747 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:27,747 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:27,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:27,747 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:35:27,747 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:35:27,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:27,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:27,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:27,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:27,749 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:35:27,749 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@285cabcd[Count = 0] remaining members to acquire global barrier 2023-10-02T12:35:27,749 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:35:27,749 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:27,750 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:27,750 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:27,750 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:27,750 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:35:27,750 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,751 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:35:27,751 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:35:27,751 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,751 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250116339 highest: 1696250116339 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,751 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770:(num 1696250116339) roll requested 2023-10-02T12:35:27,762 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250127751, exclude list is [], retry=0 2023-10-02T12:35:27,767 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:35:27,768 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250116339 with entries=109, filesize=13.56 KB; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250127751 2023-10-02T12:35:27,769 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:35:27,769 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250116339 is not closed yet, will try archiving it next time 2023-10-02T12:35:27,769 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta:.meta(num 1696250116358) roll requested 2023-10-02T12:35:27,781 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250127769.meta, exclude list is [], retry=0 2023-10-02T12:35:27,785 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:35:27,788 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250116358.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250127769.meta 2023-10-02T12:35:27,788 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:35:27,788 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250116358.meta is not closed yet, will try archiving it next time 2023-10-02T12:35:27,791 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250116358.meta to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250116358.meta 2023-10-02T12:35:27,792 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-10-02T12:35:27,792 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250127751 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,804 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:35:27,804 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40211,1696250065770' in zk 2023-10-02T12:35:27,806 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,806 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:35:27,806 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:35:27,806 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:35:27,806 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,808 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:27,808 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:27,808 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:27,809 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:27,809 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:27,809 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,810 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:27,810 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:27,810 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40211,1696250065770': 2023-10-02T12:35:27,811 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:35:27,811 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:35:27,811 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:35:27,811 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:35:27,811 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:35:27,811 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:35:27,813 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:27,813 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:27,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:27,813 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:35:27,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:27,813 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:27,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:27,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:27,814 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:27,814 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:35:27,814 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:35:27,814 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,814 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:27,814 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:27,814 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:27,815 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:27,815 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,815 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,815 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:27,816 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:27,816 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,818 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:35:27,818 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,818 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:27,818 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:35:27,818 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:35:27,818 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:27,818 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:35:27,818 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:27,818 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:27,818 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:27,818 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-10-02T12:35:27,818 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:27,818 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:35:27,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:35:27,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:35:27,820 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-10-02T12:35:27,820 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:35:27,820 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:35:27,820 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:35:27,920 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:35:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:35:27,924 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:40211=1696250105804} newestTimestamps: {jenkins-hbase3.apache.org:40211=1696250116339} 2023-10-02T12:35:27,927 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250069157.meta 2023-10-02T12:35:27,927 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250127769.meta 2023-10-02T12:35:27,927 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250116339 2023-10-02T12:35:27,927 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250127751 2023-10-02T12:35:27,928 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250085307.meta 2023-10-02T12:35:27,928 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250097907.meta 2023-10-02T12:35:27,928 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250105829.meta 2023-10-02T12:35:27,928 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250116358.meta 2023-10-02T12:35:27,985 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.15384615384615385, tune throughput to 57.69 MB/second 2023-10-02T12:35:28,340 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(146): Attempting to copy table info for:test-1696250076586 target: hdfs://localhost:33727/backupUT/backup_1696250123563/default/test-1696250076586 descriptor: 'test-1696250076586', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:35:28,340 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-10-02T12:35:28,342 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(151): Starting to write region info for table test-1696250076586 2023-10-02T12:35:28,749 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(158): Finished writing region info for table test-1696250076586 2023-10-02T12:35:29,160 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16962500765861 target: hdfs://localhost:33727/backupUT/backup_1696250123563/ns2/test-16962500765861 descriptor: 'ns2:test-16962500765861', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:35:29,161 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-10-02T12:35:29,163 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16962500765861 2023-10-02T12:35:29,169 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16962500765861 2023-10-02T12:35:29,174 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:33727/backupUT/.tmp/backup_1696250123563 from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250116339 2023-10-02T12:35:29,176 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x77230f4a to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:29,195 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47c52868, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:29,197 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:35:29,198 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37964, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:35:29,200 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:35:29,200 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16962500765861 2023-10-02T12:35:29,204 DEBUG [hconnection-0x500f685f-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:29,206 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50948, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:29,207 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16962500765861: ns2:test-16962500765861; 2023-10-02T12:35:29,207 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1696250076586 2023-10-02T12:35:29,211 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1696250076586: test-1696250076586; 2023-10-02T12:35:29,211 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-10-02T12:35:29,211 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/hbase-staging/partitions_8f89c15a-94d2-452d-9283-30ebc0169b96 2023-10-02T12:35:29,619 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:29,619 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:29,620 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:29,620 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,074 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 8a7643b8cd437a145a2c65d75c6a0149 changed from -1.0 to 0.0, refreshing cache 2023-10-02T12:35:30,613 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop-8009317751209230081.jar 2023-10-02T12:35:30,614 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,614 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,615 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,615 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,615 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,616 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,616 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,616 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,617 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,618 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:35:30,618 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:35:30,619 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:35:30,619 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:35:30,619 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:35:30,620 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:35:30,620 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:35:30,620 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:35:30,621 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:35:30,621 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:35:30,622 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:35:30,622 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:35:30,623 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,623 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,623 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,624 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,624 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,624 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:30,625 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:35:30,625 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16962500765861,default:test-1696250076586 2023-10-02T12:35:30,625 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:35:30,625 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x77230f4a to 127.0.0.1:64600 2023-10-02T12:35:30,625 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:30,626 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:35:31,643 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop-478562558504483075.jar 2023-10-02T12:35:31,650 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:35:31,673 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250116339 for WAL files 2023-10-02T12:35:31,674 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250116339 2023-10-02T12:35:31,767 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250131729/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:35:31,767 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250131729/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:35:31,824 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250116339 for hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250116339 (-9223372036854775808:9223372036854775807) length:13891 2023-10-02T12:35:31,831 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-10-02T12:35:31,831 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-10-02T12:35:31,853 DEBUG [pool-243-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x60aac4cc to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:31,858 DEBUG [pool-243-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52bff9bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:31,861 DEBUG [hconnection-0x8a22394-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:31,862 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35932, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:31,864 DEBUG [pool-243-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x60aac4cc to 127.0.0.1:64600 2023-10-02T12:35:31,864 DEBUG [pool-243-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:31,865 DEBUG [pool-243-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t1100] 2023-10-02T12:35:31,865 DEBUG [pool-243-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-10-02T12:35:31,874 DEBUG [pool-243-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/backupUT/.tmp/backup_1696250123563/_temporary/0/_temporary/attempt_local1976022238_0009_r_000000_0/ns2/test-16962500765861/f/c9dfdd196c2443e185f1e5934001cc7d is 42, key is row-t1100/f:q1/1696250123460/Put/seqid=0 2023-10-02T12:35:31,876 WARN [Thread-1232 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:40211] Specified favored nodes: [jenkins-hbase3.apache.org:40211] 2023-10-02T12:35:32,804 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:33727/backupUT 2023-10-02T12:35:32,804 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1696250123563 2023-10-02T12:35:32,804 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-10-02T12:35:32,810 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:33727/backupUT/.tmp/backup_1696250123563, hdfs://localhost:33727/backupUT] 2023-10-02T12:35:32,836 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:35:33,415 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-10-02T12:35:33,417 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1696250123563 2023-10-02T12:35:33,417 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 9405 bytes copied." 2023-10-02T12:35:33,417 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1652307389_0010 completed: true true 2023-10-02T12:35:33,421 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=5689295 FILE: Number of bytes written=12282725 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=255941 HDFS: Number of bytes written=2431248 HDFS: Number of read operations=843 HDFS: Number of large read operations=0 HDFS: Number of write operations=317 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1330642944 File Input Format Counters Bytes Read=1452 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=9405 Bytes Expected=9405 Files Copied=2 DIR_COPY=4 2023-10-02T12:35:33,421 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:33727/backupUT/.tmp/backup_1696250123563 to hdfs://localhost:33727/backupUT finished. 2023-10-02T12:35:33,433 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-10-02T12:35:33,433 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250123563 2023-10-02T12:35:33,437 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250112152 to construct the dependency. 2023-10-02T12:35:33,437 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250112152 2023-10-02T12:35:33,440 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:33,440 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250112152BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,440 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:33,440 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:33,443 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:33,443 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,443 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:33,443 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:33,445 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:33,445 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,445 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-10-02T12:35:33,445 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250123563 2023-10-02T12:35:33,447 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250112152 to construct the dependency. 2023-10-02T12:35:33,447 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250112152 2023-10-02T12:35:33,449 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:33,449 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250112152BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,449 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:33,449 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:33,451 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:33,451 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,451 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:33,451 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:33,453 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:33,454 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,454 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-10-02T12:35:33,860 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250123563/.backup.manifest 2023-10-02T12:35:33,860 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250123563 2023-10-02T12:35:33,862 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250112152 to construct the dependency. 2023-10-02T12:35:33,862 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250112152 2023-10-02T12:35:33,865 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:33,866 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250112152BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,866 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:33,866 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:33,868 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:33,868 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,868 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:33,868 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:33,871 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:33,871 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,871 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-10-02T12:35:33,871 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250123563 2023-10-02T12:35:33,873 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250112152 to construct the dependency. 2023-10-02T12:35:33,873 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250112152 2023-10-02T12:35:33,876 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:33,876 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250112152BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,876 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:33,876 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:33,878 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:33,878 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,878 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:33,878 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:33,880 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:33,880 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,880 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-10-02T12:35:33,886 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250123563/.backup.manifest 2023-10-02T12:35:33,886 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250123563 2023-10-02T12:35:33,888 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250112152 to construct the dependency. 2023-10-02T12:35:33,888 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250112152 2023-10-02T12:35:33,891 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:33,891 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250112152BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,891 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:33,891 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:33,893 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:33,893 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,893 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:33,893 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:33,895 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:33,895 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:33,895 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-10-02T12:35:34,301 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250123563/.backup.manifest 2023-10-02T12:35:34,301 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(358): Backup backup_1696250123563 finished: type=INCREMENTAL,tablelist=test-1696250076586;ns2:test-16962500765861,targetRootDir=hdfs://localhost:33727/backupUT,startts=1696250127729,completets=1696250133433,bytescopied=0 2023-10-02T12:35:34,301 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:35:34,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:35:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:35:34,309 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:35:34,311 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:35:34,314 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(376): Backup backup_1696250123563 completed. 2023-10-02T12:35:34,340 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:35:34,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:35:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250134344 (current time:1696250134344). 2023-10-02T12:35:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:35:34,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:35:34,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x39bd007e to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:34,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70641c43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:34,355 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35948, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x39bd007e to 127.0.0.1:64600 2023-10-02T12:35:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:35:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=46, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:35:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 46 2023-10-02T12:35:34,359 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:35:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-10-02T12:35:34,360 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:35:34,362 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot, current working dir path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:35:34,364 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:35:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-10-02T12:35:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-10-02T12:35:34,771 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:35:34,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2}] 2023-10-02T12:35:34,773 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:34,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40211 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=47 2023-10-02T12:35:34,925 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:34,925 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(2819): Flushing 5bf5964d15ce023a31881482c13240a2 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-10-02T12:35:34,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/90ea0b75355e4572aad52dceaf6fb04e is 171, key is trslm:hdfs://localhost:33727/backupUT\x00ns2:test-16962500765861/meta:log-roll-map/1696250133425/Put/seqid=0 2023-10-02T12:35:34,940 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/90ea0b75355e4572aad52dceaf6fb04e 2023-10-02T12:35:34,952 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/10b4b4b7e5414a7ba3c03e9f03b89c4d is 360, key is session:backup_1696250123563/session:context/1696250134310/Put/seqid=0 2023-10-02T12:35:34,957 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/10b4b4b7e5414a7ba3c03e9f03b89c4d 2023-10-02T12:35:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-10-02T12:35:34,964 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/90ea0b75355e4572aad52dceaf6fb04e as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/90ea0b75355e4572aad52dceaf6fb04e 2023-10-02T12:35:34,971 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/90ea0b75355e4572aad52dceaf6fb04e, entries=4, sequenceid=61, filesize=5.7 K 2023-10-02T12:35:34,974 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/10b4b4b7e5414a7ba3c03e9f03b89c4d as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/10b4b4b7e5414a7ba3c03e9f03b89c4d 2023-10-02T12:35:34,981 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/10b4b4b7e5414a7ba3c03e9f03b89c4d, entries=2, sequenceid=61, filesize=5.3 K 2023-10-02T12:35:34,982 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for 5bf5964d15ce023a31881482c13240a2 in 57ms, sequenceid=61, compaction requested=true 2023-10-02T12:35:34,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(2520): Flush status journal for 5bf5964d15ce023a31881482c13240a2: 2023-10-02T12:35:34,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. for snapshot_backup_system completed. 2023-10-02T12:35:34,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:35:34,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:35:34,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/a17f8975f69648a7b3a6441fe977b883, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/868b0b987e454291983936f0091fe2e3, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/90ea0b75355e4572aad52dceaf6fb04e] hfiles 2023-10-02T12:35:34,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3 for snapshot=snapshot_backup_system 2023-10-02T12:35:34,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b for snapshot=snapshot_backup_system 2023-10-02T12:35:34,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/a17f8975f69648a7b3a6441fe977b883 for snapshot=snapshot_backup_system 2023-10-02T12:35:34,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/868b0b987e454291983936f0091fe2e3 for snapshot=snapshot_backup_system 2023-10-02T12:35:34,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/90ea0b75355e4572aad52dceaf6fb04e for snapshot=snapshot_backup_system 2023-10-02T12:35:34,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/59914df0bc484059b34c6b7acf8ae537, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ffabe0f68c9244c6936357c580309e5b, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/10b4b4b7e5414a7ba3c03e9f03b89c4d] hfiles 2023-10-02T12:35:34,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10 for snapshot=snapshot_backup_system 2023-10-02T12:35:34,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46 for snapshot=snapshot_backup_system 2023-10-02T12:35:34,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7 for snapshot=snapshot_backup_system 2023-10-02T12:35:34,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/59914df0bc484059b34c6b7acf8ae537 for snapshot=snapshot_backup_system 2023-10-02T12:35:34,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ffabe0f68c9244c6936357c580309e5b for snapshot=snapshot_backup_system 2023-10-02T12:35:34,983 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/10b4b4b7e5414a7ba3c03e9f03b89c4d for snapshot=snapshot_backup_system 2023-10-02T12:35:34,990 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:34,990 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=47 2023-10-02T12:35:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster(3995): Remote procedure done, pid=47 2023-10-02T12:35:34,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:34,992 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:34,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=47, resume processing ppid=46 2023-10-02T12:35:34,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, ppid=46, state=SUCCESS; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 in 222 msec 2023-10-02T12:35:34,995 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:35:34,995 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:35:34,996 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:35:34,996 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:35:34,997 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:35:35,019 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:35:35,034 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:35:35,035 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:35:35,036 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:35:35,036 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 46 2023-10-02T12:35:35,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 679 msec 2023-10-02T12:35:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-10-02T12:35:35,465 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 46 completed 2023-10-02T12:35:35,465 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(115): Backup backup_1696250134327 started at 1696250135465. 2023-10-02T12:35:35,467 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(119): Backup session backup_1696250134327 has been started. 2023-10-02T12:35:35,472 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:35:35,473 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:35:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-10-02T12:35:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:35:35,473 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:35:35,473 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:35:35,474 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:35:35,475 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:35:35,475 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:35,475 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:35,485 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:35:35,485 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,486 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:35:35,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:35:35,486 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,486 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:35:35,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:35,486 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:35,487 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:35:35,487 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:35,487 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:35:35,487 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:35:35,487 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-10-02T12:35:35,487 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:35:35,487 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:35:35,487 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:35:35,488 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:35:35,488 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:35:35,488 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:35:35,490 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:35,490 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,490 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,490 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:35,490 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:35,491 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:35:35,491 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:35:35,491 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:35,491 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:35,492 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:35,492 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,493 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:35,493 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:35:35,493 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2c1e5000[Count = 0] remaining members to acquire global barrier 2023-10-02T12:35:35,493 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:35:35,493 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:35,495 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:35,495 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:35,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:35,495 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:35:35,495 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,495 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:35:35,495 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:35:35,495 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,495 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250127751 highest: 1696250127751 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,495 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770:(num 1696250127751) roll requested 2023-10-02T12:35:35,509 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250135495, exclude list is [], retry=0 2023-10-02T12:35:35,515 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:35:35,517 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250127751 with entries=10, filesize=2.51 KB; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250135495 2023-10-02T12:35:35,517 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:35:35,517 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250127751 is not closed yet, will try archiving it next time 2023-10-02T12:35:35,518 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta:.meta(num 1696250127769) roll requested 2023-10-02T12:35:35,534 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250135518.meta, exclude list is [], retry=0 2023-10-02T12:35:35,538 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:35:35,540 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250127769.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250135518.meta 2023-10-02T12:35:35,540 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:35:35,540 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250127769.meta is not closed yet, will try archiving it next time 2023-10-02T12:35:35,543 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250127769.meta not finished, retry = 0 2023-10-02T12:35:35,555 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-10-02T12:35:35,556 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250135495 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,565 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:35:35,565 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40211,1696250065770' in zk 2023-10-02T12:35:35,567 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:35:35,567 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:35:35,567 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:35:35,567 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,568 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,568 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:35,568 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:35,569 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:35,569 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:35,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:35,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,571 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:35,572 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:35,572 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,573 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40211,1696250065770': 2023-10-02T12:35:35,573 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:35:35,573 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:35:35,573 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:35:35,573 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:35:35,573 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:35:35,573 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:35:35,577 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:35,577 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:35:35,577 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:35,577 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:35,578 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:35:35,578 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:35:35,578 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:35,578 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:35,578 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:35,578 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:35,578 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:35,579 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:35,579 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:35,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:35,581 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:35,581 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,582 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:35,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:35,583 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,587 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:35:35,587 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,587 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:35:35,587 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:35,587 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:35,587 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:35:35,587 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:35,587 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:35:35,587 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:35,587 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:35,587 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:35,587 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:35:35,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:35:35,587 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:35:35,589 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:35:35,589 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:35:35,589 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-10-02T12:35:35,589 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:35:35,644 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250127769.meta to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250127769.meta 2023-10-02T12:35:35,689 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:35:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:35:35,694 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 } 2023-10-02T12:35:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250135694 (current time:1696250135694). 2023-10-02T12:35:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:35:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696250135693_ns3_test-16962500765862 VERSION not specified, setting to 2 2023-10-02T12:35:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x64f9bc39 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3596fe28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:35,711 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35960, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x64f9bc39 to 127.0.0.1:64600 2023-10-02T12:35:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:35:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 } 2023-10-02T12:35:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-10-02T12:35:35,716 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:35:35,717 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:35:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:35:35,723 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot, current working dir path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_1696250135693_ns3_test-16962500765862 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:35:35,726 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:35:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:35:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:35:36,133 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:35:36,133 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure 66825b0933bd359e9ecffdc9dc0cef0e}] 2023-10-02T12:35:36,135 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure 66825b0933bd359e9ecffdc9dc0cef0e 2023-10-02T12:35:36,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40211 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=49 2023-10-02T12:35:36,287 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:35:36,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2520): Flush status journal for 66825b0933bd359e9ecffdc9dc0cef0e: 2023-10-02T12:35:36,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. for snapshot_1696250135693_ns3_test-16962500765862 completed. 2023-10-02T12:35:36,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e.' region-info for snapshot=snapshot_1696250135693_ns3_test-16962500765862 2023-10-02T12:35:36,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:35:36,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:35:36,296 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:35:36,296 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=49 2023-10-02T12:35:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster(3995): Remote procedure done, pid=49 2023-10-02T12:35:36,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696250135693_ns3_test-16962500765862 on region 66825b0933bd359e9ecffdc9dc0cef0e 2023-10-02T12:35:36,298 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure 66825b0933bd359e9ecffdc9dc0cef0e 2023-10-02T12:35:36,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-10-02T12:35:36,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; SnapshotRegionProcedure 66825b0933bd359e9ecffdc9dc0cef0e in 165 msec 2023-10-02T12:35:36,305 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:35:36,306 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:35:36,306 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:35:36,306 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696250135693_ns3_test-16962500765862 2023-10-02T12:35:36,307 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_1696250135693_ns3_test-16962500765862 2023-10-02T12:35:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:35:36,720 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:35:36,727 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:35:36,727 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_1696250135693_ns3_test-16962500765862 to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_1696250135693_ns3_test-16962500765862 2023-10-02T12:35:36,729 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:35:36,729 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-10-02T12:35:36,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1696250135693_ns3_test-16962500765862 table=ns3:test-16962500765862 type=FLUSH ttl=0 } in 1.0160 sec 2023-10-02T12:35:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:35:36,825 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: ns3:test-16962500765862, procId: 48 completed 2023-10-02T12:35:36,825 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696250134327 2023-10-02T12:35:36,826 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:35:36,826 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:35:36,826 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696250134327_ns3:test-16962500765862 2023-10-02T12:35:36,826 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696250135693_ns3_test-16962500765862 to hdfs://localhost:33727/backupUT/backup_1696250134327/ns3/test-16962500765862/ 2023-10-02T12:35:36,826 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:35:36,855 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:33727, inputRoot=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3 2023-10-02T12:35:36,855 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,39919,1696250062021_-640006829_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:33727/backupUT/backup_1696250134327/ns3/test-16962500765862, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:33727/backupUT/backup_1696250134327/ns3/test-16962500765862/.hbase-snapshot/.tmp/snapshot_1696250135693_ns3_test-16962500765862 2023-10-02T12:35:36,855 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:33727, inputRoot=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3, snapshotDir=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_1696250135693_ns3_test-16962500765862. 2023-10-02T12:35:36,864 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_1696250135693_ns3_test-16962500765862 to hdfs://localhost:33727/backupUT/backup_1696250134327/ns3/test-16962500765862/.hbase-snapshot/.tmp/snapshot_1696250135693_ns3_test-16962500765862 2023-10-02T12:35:37,286 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:37,286 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:37,287 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:37,287 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:38,413 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop-1153771679413153116.jar 2023-10-02T12:35:38,414 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:38,414 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:38,414 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:38,415 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:38,415 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:38,415 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:38,416 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:38,416 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:38,416 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:38,417 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:35:38,418 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:35:38,418 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:35:38,419 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:35:38,419 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:35:38,420 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:35:38,420 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:35:38,421 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:35:38,421 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:35:38,422 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:35:38,422 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:35:38,422 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:35:38,423 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:35:38,424 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:35:38,424 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:35:38,425 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:35:38,425 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:35:38,426 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:35:38,426 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:35:38,455 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:35:38,483 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696250135693_ns3_test-16962500765862' hfile list 2023-10-02T12:35:38,627 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250138576/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:35:38,627 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250138576/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:35:39,675 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:35:39,676 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:35:39,682 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696250135693_ns3_test-16962500765862 2023-10-02T12:35:39,682 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696250135693_ns3_test-16962500765862 finished. 2023-10-02T12:35:39,682 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1034): ns3:test-16962500765862 2023-10-02T12:35:39,692 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250134327 2023-10-02T12:35:39,692 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:35:40,102 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250134327/.backup.manifest 2023-10-02T12:35:40,102 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(358): Backup backup_1696250134327 finished: type=FULL,tablelist=ns3:test-16962500765862,targetRootDir=hdfs://localhost:33727/backupUT,startts=1696250135465,completets=1696250139691,bytescopied=0 2023-10-02T12:35:40,102 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:35:40,102 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696250135693_ns3_test-16962500765862 2023-10-02T12:35:40,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696250135693_ns3_test-16962500765862" 2023-10-02T12:35:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696250135693_ns3_test-16962500765862 2023-10-02T12:35:40,107 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696250135693_ns3_test-16962500765862 for backup backup_1696250134327 succeeded. 2023-10-02T12:35:40,110 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:35:40,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:35:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:35:40,118 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:35:40,119 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:35:40,125 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(376): Backup backup_1696250134327 completed. 2023-10-02T12:35:40,165 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:35:40,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:35:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250140168 (current time:1696250140168). 2023-10-02T12:35:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:35:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:35:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x76c510f4 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:40,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68aa546a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:40,179 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50144, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x76c510f4 to 127.0.0.1:64600 2023-10-02T12:35:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:35:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:35:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-10-02T12:35:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:35:40,183 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:35:40,184 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:35:40,187 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot, current working dir path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:35:40,189 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:35:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:35:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:35:40,603 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:35:40,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2}] 2023-10-02T12:35:40,605 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:40,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40211 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2023-10-02T12:35:40,757 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:40,757 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing 5bf5964d15ce023a31881482c13240a2 2/2 column families, dataSize=1.14 KB heapSize=2.23 KB 2023-10-02T12:35:40,765 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/b83ce1ede3f146f5ab346419a272fc6d is 171, key is trslm:hdfs://localhost:33727/backupUT\x00ns3:test-16962500765862/meta:log-roll-map/1696250139687/Put/seqid=0 2023-10-02T12:35:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:35:41,171 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=458 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/b83ce1ede3f146f5ab346419a272fc6d 2023-10-02T12:35:41,180 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/5c3cce157d5e42a9a18fa5fb86ab5e31 is 446, key is session:backup_1696250134327/session:context/1696250140119/Put/seqid=0 2023-10-02T12:35:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:35:41,585 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=712 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/5c3cce157d5e42a9a18fa5fb86ab5e31 2023-10-02T12:35:41,591 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/b83ce1ede3f146f5ab346419a272fc6d as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/b83ce1ede3f146f5ab346419a272fc6d 2023-10-02T12:35:41,596 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/b83ce1ede3f146f5ab346419a272fc6d, entries=4, sequenceid=72, filesize=5.6 K 2023-10-02T12:35:41,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/5c3cce157d5e42a9a18fa5fb86ab5e31 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/5c3cce157d5e42a9a18fa5fb86ab5e31 2023-10-02T12:35:41,602 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/5c3cce157d5e42a9a18fa5fb86ab5e31, entries=2, sequenceid=72, filesize=5.4 K 2023-10-02T12:35:41,603 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1170, heapSize ~2.20 KB/2248, currentSize=0 B/0 for 5bf5964d15ce023a31881482c13240a2 in 846ms, sequenceid=72, compaction requested=true 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for 5bf5964d15ce023a31881482c13240a2: 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. for snapshot_backup_system completed. 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/a17f8975f69648a7b3a6441fe977b883, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/868b0b987e454291983936f0091fe2e3, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/90ea0b75355e4572aad52dceaf6fb04e, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/b83ce1ede3f146f5ab346419a272fc6d] hfiles 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3 for snapshot=snapshot_backup_system 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b for snapshot=snapshot_backup_system 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/a17f8975f69648a7b3a6441fe977b883 for snapshot=snapshot_backup_system 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/868b0b987e454291983936f0091fe2e3 for snapshot=snapshot_backup_system 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/90ea0b75355e4572aad52dceaf6fb04e for snapshot=snapshot_backup_system 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/b83ce1ede3f146f5ab346419a272fc6d for snapshot=snapshot_backup_system 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/59914df0bc484059b34c6b7acf8ae537, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ffabe0f68c9244c6936357c580309e5b, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/10b4b4b7e5414a7ba3c03e9f03b89c4d, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/5c3cce157d5e42a9a18fa5fb86ab5e31] hfiles 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10 for snapshot=snapshot_backup_system 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46 for snapshot=snapshot_backup_system 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7 for snapshot=snapshot_backup_system 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/59914df0bc484059b34c6b7acf8ae537 for snapshot=snapshot_backup_system 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ffabe0f68c9244c6936357c580309e5b for snapshot=snapshot_backup_system 2023-10-02T12:35:41,603 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/10b4b4b7e5414a7ba3c03e9f03b89c4d for snapshot=snapshot_backup_system 2023-10-02T12:35:41,604 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/5c3cce157d5e42a9a18fa5fb86ab5e31 for snapshot=snapshot_backup_system 2023-10-02T12:35:41,609 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:41,609 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-10-02T12:35:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster(3995): Remote procedure done, pid=51 2023-10-02T12:35:41,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:41,611 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:41,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-10-02T12:35:41,613 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:35:41,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 in 1.0080 sec 2023-10-02T12:35:41,613 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:35:41,613 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:35:41,614 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:35:41,614 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:35:42,024 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:35:42,035 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:35:42,036 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:35:42,037 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:35:42,037 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-10-02T12:35:42,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8560 sec 2023-10-02T12:35:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:35:42,290 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 50 completed 2023-10-02T12:35:42,290 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(115): Backup backup_1696250140137 started at 1696250142290. 2023-10-02T12:35:42,292 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(119): Backup session backup_1696250140137 has been started. 2023-10-02T12:35:42,297 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [ns2:test-16962500765861, ns3:test-16962500765862, test-1696250076586] 2023-10-02T12:35:42,300 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(76): StartCode 1696250116339for backupID backup_1696250140137 2023-10-02T12:35:42,300 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-10-02T12:35:42,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:35:42,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:35:42,302 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:35:42,302 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:35:42,303 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:35:42,303 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:35:42,303 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:42,303 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:42,305 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:35:42,305 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,305 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:35:42,305 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:35:42,305 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,305 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:35:42,305 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:42,306 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:42,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:35:42,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:42,306 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:35:42,306 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:35:42,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:35:42,306 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:35:42,306 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:35:42,307 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:35:42,307 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:35:42,307 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:35:42,309 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,309 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:42,309 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:42,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:42,310 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:35:42,310 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:35:42,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:42,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:42,311 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:42,311 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,311 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:42,312 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40211,1696250065770' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:35:42,312 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4ffd19f9[Count = 0] remaining members to acquire global barrier 2023-10-02T12:35:42,312 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:35:42,312 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:42,313 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:42,313 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:42,313 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:42,313 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:35:42,314 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,314 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:35:42,314 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:35:42,314 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,314 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250135495 highest: 1696250135495 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,314 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770:(num 1696250135495) roll requested 2023-10-02T12:35:42,325 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250142314, exclude list is [], retry=0 2023-10-02T12:35:42,330 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:35:42,332 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250135495 with entries=10, filesize=2.37 KB; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250142314 2023-10-02T12:35:42,332 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:35:42,332 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250135495 is not closed yet, will try archiving it next time 2023-10-02T12:35:42,332 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta:.meta(num 1696250135518) roll requested 2023-10-02T12:35:42,332 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250127751 to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250127751 2023-10-02T12:35:42,344 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250142332.meta, exclude list is [], retry=0 2023-10-02T12:35:42,348 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK] 2023-10-02T12:35:42,350 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250135518.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250142332.meta 2023-10-02T12:35:42,350 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:36399,DS-6b753d83-936a-4b84-b000-e87f03b1a904,DISK]] 2023-10-02T12:35:42,350 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250135518.meta is not closed yet, will try archiving it next time 2023-10-02T12:35:42,353 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250135518.meta to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250135518.meta 2023-10-02T12:35:42,358 DEBUG [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 44 2023-10-02T12:35:42,358 INFO [rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250142314 on jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,366 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:35:42,366 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40211,1696250065770' in zk 2023-10-02T12:35:42,368 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,368 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:35:42,368 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:35:42,368 DEBUG [member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:35:42,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:42,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:42,370 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:42,370 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:42,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:42,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:42,371 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:42,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,372 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40211,1696250065770': 2023-10-02T12:35:42,372 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40211,1696250065770' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:35:42,372 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:35:42,372 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:35:42,372 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:35:42,373 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:35:42,373 INFO [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:35:42,374 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:42,374 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:42,374 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:35:42,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:42,374 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:42,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:35:42,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:42,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:35:42,375 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:35:42,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:35:42,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:35:42,375 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:42,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:35:42,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:35:42,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:42,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,376 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:35:42,377 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:35:42,377 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,379 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,379 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:42,379 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:35:42,379 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:35:42,379 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:42,379 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:35:42,379 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:42,379 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:35:42,379 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:35:42,379 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:35:42,379 DEBUG [(jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:35:42,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:35:42,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:35:42,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:35:42,380 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:35:42,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:35:42,380 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-10-02T12:35:42,380 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:35:42,480 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:35:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:35:42,484 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:40211=1696250116339} newestTimestamps: {jenkins-hbase3.apache.org:40211=1696250135495} 2023-10-02T12:35:42,486 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250069157.meta 2023-10-02T12:35:42,486 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250142332.meta 2023-10-02T12:35:42,486 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250135495 2023-10-02T12:35:42,486 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250142314 2023-10-02T12:35:42,487 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250085307.meta 2023-10-02T12:35:42,487 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250097907.meta 2023-10-02T12:35:42,487 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250105829.meta 2023-10-02T12:35:42,487 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250116358.meta 2023-10-02T12:35:42,487 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250127769.meta 2023-10-02T12:35:42,487 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta.1696250135518.meta 2023-10-02T12:35:42,907 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(146): Attempting to copy table info for:test-1696250076586 target: hdfs://localhost:33727/backupUT/backup_1696250140137/default/test-1696250076586 descriptor: 'test-1696250076586', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:35:42,907 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-10-02T12:35:42,909 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(151): Starting to write region info for table test-1696250076586 2023-10-02T12:35:43,314 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(158): Finished writing region info for table test-1696250076586 2023-10-02T12:35:43,725 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(146): Attempting to copy table info for:ns3:test-16962500765862 target: hdfs://localhost:33727/backupUT/backup_1696250140137/ns3/test-16962500765862 descriptor: 'ns3:test-16962500765862', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:35:43,726 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-10-02T12:35:43,727 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(151): Starting to write region info for table ns3:test-16962500765862 2023-10-02T12:35:44,133 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(158): Finished writing region info for table ns3:test-16962500765862 2023-10-02T12:35:44,544 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16962500765861 target: hdfs://localhost:33727/backupUT/backup_1696250140137/ns2/test-16962500765861 descriptor: 'ns2:test-16962500765861', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:35:44,544 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-10-02T12:35:44,546 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16962500765861 2023-10-02T12:35:44,951 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16962500765861 2023-10-02T12:35:44,957 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:33727/backupUT/.tmp/backup_1696250140137 from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250135495;hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250127751 2023-10-02T12:35:44,958 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x112371b0 to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:44,965 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed86fe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:44,966 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:35:44,968 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42970, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:35:44,971 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:35:44,971 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16962500765861 2023-10-02T12:35:44,979 DEBUG [hconnection-0x14c63fa1-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:44,981 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50152, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:44,983 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16962500765861: ns2:test-16962500765861; 2023-10-02T12:35:44,983 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns3:test-16962500765862 2023-10-02T12:35:44,984 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns3:test-16962500765862: ns3:test-16962500765862; 2023-10-02T12:35:44,985 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1696250076586 2023-10-02T12:35:44,986 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1696250076586: test-1696250076586; 2023-10-02T12:35:44,986 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(694): Configuring 3 reduce partitions to match current region count for all tables 2023-10-02T12:35:44,986 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/hbase-staging/partitions_7754f9bf-dbde-4158-8f19-b3d499298fbf 2023-10-02T12:35:45,397 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:45,397 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:45,398 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:45,398 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,390 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop-55656811371291191.jar 2023-10-02T12:35:46,390 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,391 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,391 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,391 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,392 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,392 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,392 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,393 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,393 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,394 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:35:46,394 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:35:46,395 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:35:46,395 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:35:46,396 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:35:46,396 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:35:46,396 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:35:46,397 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:35:46,397 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:35:46,398 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:35:46,398 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:35:46,399 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:35:46,399 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,399 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,400 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,400 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,400 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,401 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:35:46,401 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:35:46,401 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16962500765861,ns3:test-16962500765862,default:test-1696250076586 2023-10-02T12:35:46,401 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:35:46,402 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x112371b0 to 127.0.0.1:64600 2023-10-02T12:35:46,402 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:46,403 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:35:47,436 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop-5125924703303677152.jar 2023-10-02T12:35:47,441 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:35:47,455 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250135495 for WAL files 2023-10-02T12:35:47,456 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250135495 2023-10-02T12:35:47,456 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250127751 for WAL files 2023-10-02T12:35:47,457 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250127751 2023-10-02T12:35:47,527 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250147493/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:35:47,527 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/mapreduce.cluster.local.dir/1696250147493/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:35:47,578 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250127751 for hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250127751 (-9223372036854775808:9223372036854775807) length:2574 2023-10-02T12:35:47,581 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-10-02T12:35:47,581 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-10-02T12:35:47,607 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250135495 for hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/WALs/jenkins-hbase3.apache.org,40211,1696250065770/jenkins-hbase3.apache.org%2C40211%2C1696250065770.1696250135495 (-9223372036854775808:9223372036854775807) length:2432 2023-10-02T12:35:47,610 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-10-02T12:35:47,610 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-10-02T12:35:48,562 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:33727/backupUT 2023-10-02T12:35:48,562 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1696250140137 2023-10-02T12:35:48,562 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-10-02T12:35:48,567 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:33727/backupUT/.tmp/backup_1696250140137, hdfs://localhost:33727/backupUT] 2023-10-02T12:35:48,592 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:35:49,178 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-10-02T12:35:49,180 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1696250140137 2023-10-02T12:35:49,180 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 0 bytes copied." 2023-10-02T12:35:49,180 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local957754299_0013 completed: true true 2023-10-02T12:35:49,184 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=6749959 FILE: Number of bytes written=16006285 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=326545 HDFS: Number of bytes written=2475655 HDFS: Number of read operations=1076 HDFS: Number of large read operations=0 HDFS: Number of write operations=382 Map-Reduce Framework Map input records=2 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1340080128 File Input Format Counters Bytes Read=486 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=0 Bytes Expected=0 Files Copied=1 DIR_COPY=1 2023-10-02T12:35:49,184 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:33727/backupUT/.tmp/backup_1696250140137 to hdfs://localhost:33727/backupUT finished. 2023-10-02T12:35:49,194 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-10-02T12:35:49,194 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250140137 2023-10-02T12:35:49,199 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250123563 to construct the dependency. 2023-10-02T12:35:49,199 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250123563 2023-10-02T12:35:49,201 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250123563/.backup.manifest 2023-10-02T12:35:49,201 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250123563BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,201 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250112152 to construct the dependency. 2023-10-02T12:35:49,202 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250112152 2023-10-02T12:35:49,204 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:49,204 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250112152BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,204 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:49,204 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:49,207 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:49,207 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,207 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:49,207 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:49,209 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:49,209 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,209 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-10-02T12:35:49,209 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250140137 2023-10-02T12:35:49,211 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250123563 to construct the dependency. 2023-10-02T12:35:49,211 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250123563 2023-10-02T12:35:49,214 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250123563/.backup.manifest 2023-10-02T12:35:49,214 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250123563BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,214 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250112152 to construct the dependency. 2023-10-02T12:35:49,214 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250112152 2023-10-02T12:35:49,216 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:49,216 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250112152BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,216 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:49,216 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:49,218 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:49,219 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,219 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:49,219 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:49,221 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:49,221 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,221 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-10-02T12:35:49,628 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250140137/.backup.manifest 2023-10-02T12:35:49,628 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250140137 2023-10-02T12:35:49,631 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250123563 to construct the dependency. 2023-10-02T12:35:49,631 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250123563 2023-10-02T12:35:49,634 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250123563/.backup.manifest 2023-10-02T12:35:49,634 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250123563BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,635 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250112152 to construct the dependency. 2023-10-02T12:35:49,635 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250112152 2023-10-02T12:35:49,637 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:49,637 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250112152BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,638 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:49,638 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:49,640 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:49,640 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,640 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:49,640 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:49,642 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:49,642 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,642 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-10-02T12:35:49,642 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250140137 2023-10-02T12:35:49,644 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250123563 to construct the dependency. 2023-10-02T12:35:49,644 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250123563 2023-10-02T12:35:49,647 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250123563/.backup.manifest 2023-10-02T12:35:49,647 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250123563BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,647 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250112152 to construct the dependency. 2023-10-02T12:35:49,647 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250112152 2023-10-02T12:35:49,649 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:49,649 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250112152BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,650 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:49,650 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:49,652 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:49,652 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,652 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:49,652 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:49,654 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:49,654 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:49,654 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-10-02T12:35:50,061 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250140137/.backup.manifest 2023-10-02T12:35:50,061 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250140137 2023-10-02T12:35:50,063 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250123563 to construct the dependency. 2023-10-02T12:35:50,063 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250123563 2023-10-02T12:35:50,066 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250123563/.backup.manifest 2023-10-02T12:35:50,066 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250123563BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:50,066 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250112152 to construct the dependency. 2023-10-02T12:35:50,067 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250112152 2023-10-02T12:35:50,069 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:50,069 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250112152BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:50,069 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:50,069 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:50,071 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:50,072 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:50,072 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:50,072 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:50,074 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:50,074 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:50,074 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-10-02T12:35:50,074 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250140137 2023-10-02T12:35:50,076 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250123563 to construct the dependency. 2023-10-02T12:35:50,076 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250123563 2023-10-02T12:35:50,078 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250123563/.backup.manifest 2023-10-02T12:35:50,078 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250123563BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:50,078 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250112152 to construct the dependency. 2023-10-02T12:35:50,078 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250112152 2023-10-02T12:35:50,080 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:50,080 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250112152BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:50,080 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:50,080 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:50,082 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:50,082 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:50,082 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:50,082 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:50,084 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:50,084 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:50,084 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-10-02T12:35:50,490 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250140137/.backup.manifest 2023-10-02T12:35:50,490 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696250140137 2023-10-02T12:35:50,492 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250123563 to construct the dependency. 2023-10-02T12:35:50,492 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250123563 2023-10-02T12:35:50,495 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250123563/.backup.manifest 2023-10-02T12:35:50,495 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250123563BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:50,495 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250112152 to construct the dependency. 2023-10-02T12:35:50,495 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250112152 2023-10-02T12:35:50,497 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250112152/.backup.manifest 2023-10-02T12:35:50,497 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250112152BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:50,497 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250103611 to construct the dependency. 2023-10-02T12:35:50,498 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250103611 2023-10-02T12:35:50,499 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250103611/.backup.manifest 2023-10-02T12:35:50,499 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250103611BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:50,500 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:33727/backupUT/backup_1696250096681 to construct the dependency. 2023-10-02T12:35:50,500 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:33727/backupUT/backup_1696250096681 2023-10-02T12:35:50,502 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696250096681/.backup.manifest 2023-10-02T12:35:50,502 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1696250096681BackupDir=hdfs://localhost:33727/backupUT} 2023-10-02T12:35:50,502 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-10-02T12:35:50,908 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:33727/backupUT/backup_1696250140137/.backup.manifest 2023-10-02T12:35:50,908 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(358): Backup backup_1696250140137 finished: type=INCREMENTAL,tablelist=test-1696250076586;ns3:test-16962500765862;ns2:test-16962500765861,targetRootDir=hdfs://localhost:33727/backupUT,startts=1696250142290,completets=1696250149194,bytescopied=0 2023-10-02T12:35:50,908 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:35:50,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:35:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:35:50,916 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:35:50,917 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:35:50,920 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.TableBackupClient(376): Backup backup_1696250140137 completed. 2023-10-02T12:35:50,932 ERROR [Listener at jenkins-hbase3.apache.org/34739 {}] backup.TestBackupMultipleDeletes(130): Delete backupIdInc2 2023-10-02T12:35:50,939 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:35:50,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:35:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250150950 (current time:1696250150950). 2023-10-02T12:35:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:35:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:35:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2681fa3e to 127.0.0.1:64600 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:35:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fbe5e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:35:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:35:50,959 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35442, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:35:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2681fa3e to 127.0.0.1:64600 2023-10-02T12:35:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:35:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure2.ProcedureExecutor(1032): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:35:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-10-02T12:35:50,962 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:35:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:35:50,963 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:35:50,966 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot, current working dir path: hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:35:50,968 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:35:50,974 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:35:50,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2}] 2023-10-02T12:35:50,975 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:35:51,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40211 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-10-02T12:35:51,127 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:51,127 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing 5bf5964d15ce023a31881482c13240a2 2/2 column families, dataSize=1.75 KB heapSize=3.05 KB 2023-10-02T12:35:51,131 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/918969866d51438a881304d9edf28ff3 is 171, key is trslm:hdfs://localhost:33727/backupUT\x00ns2:test-16962500765861/meta:log-roll-map/1696250149188/Put/seqid=0 2023-10-02T12:35:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:35:51,536 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=758 B at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/918969866d51438a881304d9edf28ff3 2023-10-02T12:35:51,546 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/99f0dc8fa8ec4187a5fb6a8c43b83d16 is 461, key is session:backup_1696250140137/session:context/1696250150916/Put/seqid=0 2023-10-02T12:35:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:35:51,951 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.01 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/99f0dc8fa8ec4187a5fb6a8c43b83d16 2023-10-02T12:35:51,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/918969866d51438a881304d9edf28ff3 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/918969866d51438a881304d9edf28ff3 2023-10-02T12:35:51,962 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/918969866d51438a881304d9edf28ff3, entries=6, sequenceid=84, filesize=5.8 K 2023-10-02T12:35:51,963 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/99f0dc8fa8ec4187a5fb6a8c43b83d16 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/99f0dc8fa8ec4187a5fb6a8c43b83d16 2023-10-02T12:35:51,969 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/99f0dc8fa8ec4187a5fb6a8c43b83d16, entries=2, sequenceid=84, filesize=5.4 K 2023-10-02T12:35:51,970 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~1.75 KB/1792, heapSize ~3.02 KB/3096, currentSize=0 B/0 for 5bf5964d15ce023a31881482c13240a2 in 843ms, sequenceid=84, compaction requested=true 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for 5bf5964d15ce023a31881482c13240a2: 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. for snapshot_backup_system completed. 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/a17f8975f69648a7b3a6441fe977b883, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/868b0b987e454291983936f0091fe2e3, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/90ea0b75355e4572aad52dceaf6fb04e, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/b83ce1ede3f146f5ab346419a272fc6d, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/918969866d51438a881304d9edf28ff3] hfiles 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/534be26151d942a196491c2a2fc799b3 for snapshot=snapshot_backup_system 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/6902999acb1740f2bc2bc521b427574b for snapshot=snapshot_backup_system 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/a17f8975f69648a7b3a6441fe977b883 for snapshot=snapshot_backup_system 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/868b0b987e454291983936f0091fe2e3 for snapshot=snapshot_backup_system 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/90ea0b75355e4572aad52dceaf6fb04e for snapshot=snapshot_backup_system 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/b83ce1ede3f146f5ab346419a272fc6d for snapshot=snapshot_backup_system 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/918969866d51438a881304d9edf28ff3 for snapshot=snapshot_backup_system 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/59914df0bc484059b34c6b7acf8ae537, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ffabe0f68c9244c6936357c580309e5b, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/10b4b4b7e5414a7ba3c03e9f03b89c4d, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/5c3cce157d5e42a9a18fa5fb86ab5e31, hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/99f0dc8fa8ec4187a5fb6a8c43b83d16] hfiles 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/fcaa57170a1440eb907a0c18a525ad10 for snapshot=snapshot_backup_system 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ff8629cab718428281c86435a4858f46 for snapshot=snapshot_backup_system 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ea7b382616974833926d102b744ceab7 for snapshot=snapshot_backup_system 2023-10-02T12:35:51,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/59914df0bc484059b34c6b7acf8ae537 for snapshot=snapshot_backup_system 2023-10-02T12:35:51,971 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/ffabe0f68c9244c6936357c580309e5b for snapshot=snapshot_backup_system 2023-10-02T12:35:51,971 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/10b4b4b7e5414a7ba3c03e9f03b89c4d for snapshot=snapshot_backup_system 2023-10-02T12:35:51,971 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/5c3cce157d5e42a9a18fa5fb86ab5e31 for snapshot=snapshot_backup_system 2023-10-02T12:35:51,971 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/99f0dc8fa8ec4187a5fb6a8c43b83d16 for snapshot=snapshot_backup_system 2023-10-02T12:35:51,977 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:51,977 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-10-02T12:35:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.HMaster(3995): Remote procedure done, pid=53 2023-10-02T12:35:51,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:51,979 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:51,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-10-02T12:35:51,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure 5bf5964d15ce023a31881482c13240a2 in 1.0060 sec 2023-10-02T12:35:51,982 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:35:51,982 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:35:51,983 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:35:51,983 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:35:51,984 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:35:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:35:52,397 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:35:52,409 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:35:52,409 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:35:52,411 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:35:52,411 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-10-02T12:35:52,412 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.4510 sec 2023-10-02T12:35:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:35:53,069 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 52 completed 2023-10-02T12:35:53,074 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(227): Deleting backup backup_1696250103611 ... 2023-10-02T12:35:53,075 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-10-02T12:35:53,075 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(435): Delete log files: _SUCCESS 2023-10-02T12:35:53,076 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(435): Delete log files: default 2023-10-02T12:35:53,076 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(435): Delete log files: ns2 2023-10-02T12:35:53,077 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1696250103611 2023-10-02T12:35:53,077 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:33727/backupUT/backup_1696250103611/default/test-1696250076586. 2023-10-02T12:35:53,078 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(465): hdfs://localhost:33727/backupUT/backup_1696250103611/default is empty, remove it. 2023-10-02T12:35:53,078 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:33727/backupUT/backup_1696250103611/ns2/test-16962500765861. 2023-10-02T12:35:53,079 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(465): hdfs://localhost:33727/backupUT/backup_1696250103611/ns2 is empty, remove it. 2023-10-02T12:35:53,082 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(309): GetAffectedBackupInfos for: backup_1696250103611 table=test-1696250076586 2023-10-02T12:35:53,084 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1696250103611 table=test-1696250076586 added backup_1696250140137 tables={test-1696250076586,ns3:test-16962500765862,ns2:test-16962500765861} 2023-10-02T12:35:53,084 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1696250103611 table=test-1696250076586 added backup_1696250123563 tables={test-1696250076586,ns2:test-16962500765861} 2023-10-02T12:35:53,085 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1696250103611 table=test-1696250076586 added backup_1696250112152 tables={test-1696250076586,ns2:test-16962500765861} 2023-10-02T12:35:53,085 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(287): Remove test-1696250076586 from backup_1696250140137 tables={test-1696250076586,ns3:test-16962500765862,ns2:test-16962500765861} 2023-10-02T12:35:53,087 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:33727/backupUT/backup_1696250140137/default/test-1696250076586 done. 2023-10-02T12:35:53,087 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(287): Remove test-1696250076586 from backup_1696250123563 tables={test-1696250076586,ns2:test-16962500765861} 2023-10-02T12:35:53,089 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:33727/backupUT/backup_1696250123563/default/test-1696250076586 done. 2023-10-02T12:35:53,089 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(287): Remove test-1696250076586 from backup_1696250112152 tables={test-1696250076586,ns2:test-16962500765861} 2023-10-02T12:35:53,091 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:33727/backupUT/backup_1696250112152/default/test-1696250076586 done. 2023-10-02T12:35:53,093 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(309): GetAffectedBackupInfos for: backup_1696250103611 table=ns2:test-16962500765861 2023-10-02T12:35:53,096 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1696250103611 table=ns2:test-16962500765861 added backup_1696250140137 tables={ns3:test-16962500765862,ns2:test-16962500765861} 2023-10-02T12:35:53,096 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1696250103611 table=ns2:test-16962500765861 added backup_1696250123563 tables={ns2:test-16962500765861} 2023-10-02T12:35:53,096 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1696250103611 table=ns2:test-16962500765861 added backup_1696250112152 tables={ns2:test-16962500765861} 2023-10-02T12:35:53,096 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(287): Remove ns2:test-16962500765861 from backup_1696250140137 tables={ns3:test-16962500765862,ns2:test-16962500765861} 2023-10-02T12:35:53,098 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:33727/backupUT/backup_1696250140137/ns2/test-16962500765861 done. 2023-10-02T12:35:53,098 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(287): Remove ns2:test-16962500765861 from backup_1696250123563 tables={ns2:test-16962500765861} 2023-10-02T12:35:53,099 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(293): Delete backup info backup_1696250123563 2023-10-02T12:35:53,102 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-10-02T12:35:53,102 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(435): Delete log files: _SUCCESS 2023-10-02T12:35:53,103 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(435): Delete log files: default 2023-10-02T12:35:53,103 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(435): Delete log files: ns2 2023-10-02T12:35:53,104 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1696250123563 2023-10-02T12:35:53,104 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:33727/backupUT/backup_1696250123563/ns2/test-16962500765861. 2023-10-02T12:35:53,105 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(465): hdfs://localhost:33727/backupUT/backup_1696250123563/ns2 is empty, remove it. 2023-10-02T12:35:53,106 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(287): Remove ns2:test-16962500765861 from backup_1696250112152 tables={ns2:test-16962500765861} 2023-10-02T12:35:53,106 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(293): Delete backup info backup_1696250112152 2023-10-02T12:35:53,108 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-10-02T12:35:53,108 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(435): Delete log files: _SUCCESS 2023-10-02T12:35:53,109 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(435): Delete log files: default 2023-10-02T12:35:53,109 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(435): Delete log files: ns2 2023-10-02T12:35:53,110 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1696250112152 2023-10-02T12:35:53,110 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:33727/backupUT/backup_1696250112152/ns2/test-16962500765861. 2023-10-02T12:35:53,111 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.BackupUtils(465): hdfs://localhost:33727/backupUT/backup_1696250112152/ns2 is empty, remove it. 2023-10-02T12:35:53,113 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-10-02T12:35:53,113 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-10-02T12:35:53,114 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupAdminImpl(276): Delete backup backup_1696250103611 completed. 2023-10-02T12:35:53,120 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1034): ns2:test-16962500765861 2023-10-02T12:35:53,120 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1034): ns3:test-16962500765862 2023-10-02T12:35:53,120 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1034): test-1696250076586 2023-10-02T12:35:53,122 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:35:53,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:35:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39919 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:35:53,128 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:35:53,128 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:35:53,130 ERROR [Listener at jenkins-hbase3.apache.org/34739 {}] backup.TestBackupMultipleDeletes(132): Delete backupIdInc2 done 2023-10-02T12:35:53,140 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:35:53,140 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0842c64b to 127.0.0.1:64600 2023-10-02T12:35:53,141 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:53,165 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.ResourceChecker(175): after: backup.TestBackupMultipleDeletes#testBackupMultipleDeletes Thread=594 (was 584) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_358399394_16 at /127.0.0.1:37290 [Waiting for operation #9] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x40922443-shared-pool-29 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x40922443-shared-pool-30 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1578325291-172.31.12.81-1696250058286:blk_1073741959_1135, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/cluster_9dfd02cb-0bbd-743a-fb9e-a562fae510de/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x72683eba-shared-pool-15 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575232512_16 at /127.0.0.1:37272 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1575232512_16 at /127.0.0.1:52874 [Waiting for operation #45] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,40211,1696250065770' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x40922443-shared-pool-31 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x72683eba-shared-pool-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/cluster_9dfd02cb-0bbd-743a-fb9e-a562fae510de/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1578325291-172.31.12.81-1696250058286:blk_1073741960_1136, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,39919,1696250062021)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,40211,1696250065770)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-586639559_16 at /127.0.0.1:52794 [Receiving block BP-1578325291-172.31.12.81-1696250058286:blk_1073741959_1135] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-586639559_16 at /127.0.0.1:52796 [Receiving block BP-1578325291-172.31.12.81-1696250058286:blk_1073741960_1136] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=677 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=389 (was 558), ProcessCount=181 (was 181), AvailableMemoryMB=1561 (was 3175) 2023-10-02T12:35:53,165 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.ResourceChecker(130): Thread=594 is superior to 500 2023-10-02T12:35:53,172 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-10-02T12:35:53,172 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:35:53,173 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x10da4c22 to 127.0.0.1:64600 2023-10-02T12:35:53,173 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:53,173 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-10-02T12:35:53,173 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] util.JVMClusterUtil(257): Found active master hash=1753030297, stopped=false 2023-10-02T12:35:53,173 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,39919,1696250062021 2023-10-02T12:35:53,176 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-10-02T12:35:53,176 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:35:53,176 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] procedure2.ProcedureExecutor(630): Stopping 2023-10-02T12:35:53,177 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-10-02T12:35:53,177 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:35:53,177 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:35:53,177 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:35:53,177 DEBUG [Listener at jenkins-hbase3.apache.org/34739 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:53,178 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,40211,1696250065770' ***** 2023-10-02T12:35:53,178 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-10-02T12:35:53,178 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:35:53,194 INFO [RS:0;jenkins-hbase3:40211 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@245e1c35{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:35:53,199 INFO [RS:0;jenkins-hbase3:40211 {}] server.AbstractConnector(383): Stopped ServerConnector@4b8b4081{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:35:53,199 INFO [RS:0;jenkins-hbase3:40211 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:35:53,200 INFO [RS:0;jenkins-hbase3:40211 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@640bb327{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:35:53,201 INFO [RS:0;jenkins-hbase3:40211 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@4a19e3c9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop.log.dir/,STOPPED} 2023-10-02T12:35:53,203 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HeapMemoryManager(220): Stopping 2023-10-02T12:35:53,204 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-10-02T12:35:53,204 INFO [RS:0;jenkins-hbase3:40211 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-10-02T12:35:53,204 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-10-02T12:35:53,204 INFO [RS:0;jenkins-hbase3:40211 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-10-02T12:35:53,204 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(3528): Received CLOSE for 97d4833ae9b729f41ce094e6b5f5d9e0 2023-10-02T12:35:53,205 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(3528): Received CLOSE for 66825b0933bd359e9ecffdc9dc0cef0e 2023-10-02T12:35:53,205 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(3528): Received CLOSE for f846b5ddba88af6ce438c513542c180e 2023-10-02T12:35:53,205 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(3528): Received CLOSE for 04056acb44d8ef0206796c62cd889bed 2023-10-02T12:35:53,205 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(3528): Received CLOSE for 8a7643b8cd437a145a2c65d75c6a0149 2023-10-02T12:35:53,205 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(3528): Received CLOSE for 5bf5964d15ce023a31881482c13240a2 2023-10-02T12:35:53,205 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(3528): Received CLOSE for ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:35:53,205 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:53,205 INFO [RS:0;jenkins-hbase3:40211 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:35:53,205 DEBUG [RS:0;jenkins-hbase3:40211 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:53,205 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 97d4833ae9b729f41ce094e6b5f5d9e0, disabling compactions & flushes 2023-10-02T12:35:53,205 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:35:53,205 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-10-02T12:35:53,205 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:35:53,205 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-10-02T12:35:53,205 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. after waiting 0 ms 2023-10-02T12:35:53,205 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-10-02T12:35:53,205 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:35:53,205 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-10-02T12:35:53,206 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 97d4833ae9b729f41ce094e6b5f5d9e0 1/1 column families, dataSize=7.13 KB heapSize=21.91 KB 2023-10-02T12:35:53,206 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-10-02T12:35:53,207 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1583): Online Regions={97d4833ae9b729f41ce094e6b5f5d9e0=ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0., 66825b0933bd359e9ecffdc9dc0cef0e=ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e., f846b5ddba88af6ce438c513542c180e=backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e., 1588230740=hbase:meta,,1.1588230740, 04056acb44d8ef0206796c62cd889bed=test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed., 8a7643b8cd437a145a2c65d75c6a0149=hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149., 5bf5964d15ce023a31881482c13240a2=backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2., ee1a09a86970bc5e288b8a3080cb4647=ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647.} 2023-10-02T12:35:53,207 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:35:53,207 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:35:53,207 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:35:53,207 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:35:53,207 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:35:53,207 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=9.25 KB heapSize=16.20 KB 2023-10-02T12:35:53,211 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1609): Waiting on 04056acb44d8ef0206796c62cd889bed, 1588230740, 5bf5964d15ce023a31881482c13240a2, 66825b0933bd359e9ecffdc9dc0cef0e, 8a7643b8cd437a145a2c65d75c6a0149, 97d4833ae9b729f41ce094e6b5f5d9e0, ee1a09a86970bc5e288b8a3080cb4647, f846b5ddba88af6ce438c513542c180e 2023-10-02T12:35:53,220 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/.tmp/f/3b8dbcb937ba40c8931bed6bf0666faf is 42, key is row-t1100/f:q1/1696250123460/Put/seqid=0 2023-10-02T12:35:53,231 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/.tmp/f/3b8dbcb937ba40c8931bed6bf0666faf 2023-10-02T12:35:53,239 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/.tmp/f/3b8dbcb937ba40c8931bed6bf0666faf as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/f/3b8dbcb937ba40c8931bed6bf0666faf 2023-10-02T12:35:53,240 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740/.tmp/info/1d6a7261e7344bafa0d664d76ba65477 is 159, key is ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0./info:regioninfo/1696250079280/Put/seqid=0 2023-10-02T12:35:53,246 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/f/3b8dbcb937ba40c8931bed6bf0666faf, entries=198, sequenceid=304, filesize=12.6 K 2023-10-02T12:35:53,247 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~7.13 KB/7304, heapSize ~21.89 KB/22416, currentSize=0 B/0 for 97d4833ae9b729f41ce094e6b5f5d9e0 in 42ms, sequenceid=304, compaction requested=false 2023-10-02T12:35:53,260 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740/.tmp/info/1d6a7261e7344bafa0d664d76ba65477 2023-10-02T12:35:53,263 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns2/test-16962500765861/97d4833ae9b729f41ce094e6b5f5d9e0/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=1 2023-10-02T12:35:53,263 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:35:53,264 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:35:53,264 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 97d4833ae9b729f41ce094e6b5f5d9e0: 2023-10-02T12:35:53,265 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16962500765861,,1696250078459.97d4833ae9b729f41ce094e6b5f5d9e0. 2023-10-02T12:35:53,265 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 66825b0933bd359e9ecffdc9dc0cef0e, disabling compactions & flushes 2023-10-02T12:35:53,265 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:35:53,265 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:35:53,265 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. after waiting 0 ms 2023-10-02T12:35:53,265 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:35:53,269 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns3/test-16962500765862/66825b0933bd359e9ecffdc9dc0cef0e/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2023-10-02T12:35:53,269 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:35:53,270 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:35:53,270 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 66825b0933bd359e9ecffdc9dc0cef0e: 2023-10-02T12:35:53,270 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16962500765862,,1696250079698.66825b0933bd359e9ecffdc9dc0cef0e. 2023-10-02T12:35:53,271 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing f846b5ddba88af6ce438c513542c180e, disabling compactions & flushes 2023-10-02T12:35:53,271 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. 2023-10-02T12:35:53,271 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. 2023-10-02T12:35:53,271 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. after waiting 0 ms 2023-10-02T12:35:53,271 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. 2023-10-02T12:35:53,272 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:35:53,274 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system_bulk/f846b5ddba88af6ce438c513542c180e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:35:53,274 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:35:53,275 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. 2023-10-02T12:35:53,275 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for f846b5ddba88af6ce438c513542c180e: 2023-10-02T12:35:53,276 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1696250082796.f846b5ddba88af6ce438c513542c180e. 2023-10-02T12:35:53,276 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 04056acb44d8ef0206796c62cd889bed, disabling compactions & flushes 2023-10-02T12:35:53,276 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:35:53,276 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:35:53,276 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. after waiting 0 ms 2023-10-02T12:35:53,276 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:35:53,276 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 04056acb44d8ef0206796c62cd889bed 1/1 column families, dataSize=7.13 KB heapSize=21.91 KB 2023-10-02T12:35:53,280 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/.tmp/f/c6426a7d99944c51ae0b80ce71887e27 is 42, key is row-t1100/f:q1/1696250112041/Put/seqid=0 2023-10-02T12:35:53,282 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740/.tmp/table/e2284fe8781f418581f69720894f82cc is 59, key is ns2:test-16962500765861/table:state/1696250079292/Put/seqid=0 2023-10-02T12:35:53,348 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-10-02T12:35:53,349 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962500765861 2023-10-02T12:35:53,350 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962500765862 2023-10-02T12:35:53,411 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1609): Waiting on 04056acb44d8ef0206796c62cd889bed, 1588230740, 5bf5964d15ce023a31881482c13240a2, 8a7643b8cd437a145a2c65d75c6a0149, ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:35:53,611 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1609): Waiting on 04056acb44d8ef0206796c62cd889bed, 1588230740, 5bf5964d15ce023a31881482c13240a2, 8a7643b8cd437a145a2c65d75c6a0149, ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:35:53,688 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/.tmp/f/c6426a7d99944c51ae0b80ce71887e27 2023-10-02T12:35:53,689 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=714 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740/.tmp/table/e2284fe8781f418581f69720894f82cc 2023-10-02T12:35:53,695 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/.tmp/f/c6426a7d99944c51ae0b80ce71887e27 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/f/c6426a7d99944c51ae0b80ce71887e27 2023-10-02T12:35:53,696 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740/.tmp/info/1d6a7261e7344bafa0d664d76ba65477 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740/info/1d6a7261e7344bafa0d664d76ba65477 2023-10-02T12:35:53,702 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/f/c6426a7d99944c51ae0b80ce71887e27, entries=198, sequenceid=304, filesize=12.6 K 2023-10-02T12:35:53,702 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740/info/1d6a7261e7344bafa0d664d76ba65477, entries=70, sequenceid=39, filesize=14.1 K 2023-10-02T12:35:53,703 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~7.13 KB/7304, heapSize ~21.89 KB/22416, currentSize=0 B/0 for 04056acb44d8ef0206796c62cd889bed in 427ms, sequenceid=304, compaction requested=false 2023-10-02T12:35:53,704 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740/.tmp/table/e2284fe8781f418581f69720894f82cc as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740/table/e2284fe8781f418581f69720894f82cc 2023-10-02T12:35:53,723 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/default/test-1696250076586/04056acb44d8ef0206796c62cd889bed/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=1 2023-10-02T12:35:53,724 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:35:53,725 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:35:53,725 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 04056acb44d8ef0206796c62cd889bed: 2023-10-02T12:35:53,725 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1696250076586,,1696250077109.04056acb44d8ef0206796c62cd889bed. 2023-10-02T12:35:53,726 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 8a7643b8cd437a145a2c65d75c6a0149, disabling compactions & flushes 2023-10-02T12:35:53,726 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. 2023-10-02T12:35:53,726 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. 2023-10-02T12:35:53,726 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. after waiting 0 ms 2023-10-02T12:35:53,726 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. 2023-10-02T12:35:53,726 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 8a7643b8cd437a145a2c65d75c6a0149 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-10-02T12:35:53,729 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740/table/e2284fe8781f418581f69720894f82cc, entries=14, sequenceid=39, filesize=5.8 K 2023-10-02T12:35:53,730 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~9.25 KB/9474, heapSize ~15.92 KB/16304, currentSize=0 B/0 for 1588230740 in 522ms, sequenceid=39, compaction requested=false 2023-10-02T12:35:53,742 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/meta/1588230740/recovered.edits/42.seqid, newMaxSeqId=42, maxSeqId=1 2023-10-02T12:35:53,742 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:35:53,742 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:35:53,743 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:35:53,743 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:35:53,743 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-10-02T12:35:53,747 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/namespace/8a7643b8cd437a145a2c65d75c6a0149/.tmp/info/8bbba29ca73d42d2afb035de65dc7287 is 45, key is default/info:d/1696250070474/Put/seqid=0 2023-10-02T12:35:53,812 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1609): Waiting on 5bf5964d15ce023a31881482c13240a2, 8a7643b8cd437a145a2c65d75c6a0149, ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:35:54,012 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1609): Waiting on 5bf5964d15ce023a31881482c13240a2, 8a7643b8cd437a145a2c65d75c6a0149, ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:35:54,071 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-10-02T12:35:54,071 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-10-02T12:35:54,154 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/namespace/8a7643b8cd437a145a2c65d75c6a0149/.tmp/info/8bbba29ca73d42d2afb035de65dc7287 2023-10-02T12:35:54,161 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/namespace/8a7643b8cd437a145a2c65d75c6a0149/.tmp/info/8bbba29ca73d42d2afb035de65dc7287 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/namespace/8a7643b8cd437a145a2c65d75c6a0149/info/8bbba29ca73d42d2afb035de65dc7287 2023-10-02T12:35:54,166 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/namespace/8a7643b8cd437a145a2c65d75c6a0149/info/8bbba29ca73d42d2afb035de65dc7287, entries=7, sequenceid=11, filesize=5.1 K 2023-10-02T12:35:54,167 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 8a7643b8cd437a145a2c65d75c6a0149 in 441ms, sequenceid=11, compaction requested=false 2023-10-02T12:35:54,174 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/hbase/namespace/8a7643b8cd437a145a2c65d75c6a0149/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-10-02T12:35:54,175 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:35:54,175 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. 2023-10-02T12:35:54,175 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 8a7643b8cd437a145a2c65d75c6a0149: 2023-10-02T12:35:54,175 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1696250069737.8a7643b8cd437a145a2c65d75c6a0149. 2023-10-02T12:35:54,175 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 5bf5964d15ce023a31881482c13240a2, disabling compactions & flushes 2023-10-02T12:35:54,175 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:54,175 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:54,176 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. after waiting 0 ms 2023-10-02T12:35:54,176 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:54,176 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 5bf5964d15ce023a31881482c13240a2 2/2 column families, dataSize=1.93 KB heapSize=3.31 KB 2023-10-02T12:35:54,179 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/aee4715bddaa4c8b96502afeaade99c5 is 96, key is incrbackupset:hdfs://localhost:33727/backupUT/meta:ns2:test-16962500765861/1696250153120/Put/seqid=0 2023-10-02T12:35:54,212 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-10-02T12:35:54,212 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1583): Online Regions={5bf5964d15ce023a31881482c13240a2=backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2., ee1a09a86970bc5e288b8a3080cb4647=ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647.} 2023-10-02T12:35:54,212 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1609): Waiting on 5bf5964d15ce023a31881482c13240a2, ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:35:54,413 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1609): Waiting on 5bf5964d15ce023a31881482c13240a2, ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:35:54,584 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=308 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/aee4715bddaa4c8b96502afeaade99c5 2023-10-02T12:35:54,590 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aee4715bddaa4c8b96502afeaade99c5 2023-10-02T12:35:54,594 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/62b3c14762ab499f92a7419acb680c2d is 398, key is session:backup_1696250140137/session:context/1696250153097/Put/seqid=0 2023-10-02T12:35:54,613 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1609): Waiting on 5bf5964d15ce023a31881482c13240a2, ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:35:54,813 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1609): Waiting on 5bf5964d15ce023a31881482c13240a2, ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:35:54,999 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.63 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/62b3c14762ab499f92a7419acb680c2d 2023-10-02T12:35:55,004 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 62b3c14762ab499f92a7419acb680c2d 2023-10-02T12:35:55,005 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/meta/aee4715bddaa4c8b96502afeaade99c5 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/aee4715bddaa4c8b96502afeaade99c5 2023-10-02T12:35:55,010 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aee4715bddaa4c8b96502afeaade99c5 2023-10-02T12:35:55,010 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/meta/aee4715bddaa4c8b96502afeaade99c5, entries=4, sequenceid=97, filesize=5.4 K 2023-10-02T12:35:55,011 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/.tmp/session/62b3c14762ab499f92a7419acb680c2d as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/62b3c14762ab499f92a7419acb680c2d 2023-10-02T12:35:55,013 DEBUG [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1609): Waiting on 5bf5964d15ce023a31881482c13240a2, ee1a09a86970bc5e288b8a3080cb4647 2023-10-02T12:35:55,016 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 62b3c14762ab499f92a7419acb680c2d 2023-10-02T12:35:55,016 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/session/62b3c14762ab499f92a7419acb680c2d, entries=5, sequenceid=97, filesize=5.7 K 2023-10-02T12:35:55,017 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~1.93 KB/1981, heapSize ~3.28 KB/3360, currentSize=0 B/0 for 5bf5964d15ce023a31881482c13240a2 in 841ms, sequenceid=97, compaction requested=true 2023-10-02T12:35:55,046 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/backup/system/5bf5964d15ce023a31881482c13240a2/recovered.edits/100.seqid, newMaxSeqId=100, maxSeqId=1 2023-10-02T12:35:55,047 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:35:55,048 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:55,048 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 5bf5964d15ce023a31881482c13240a2: 2023-10-02T12:35:55,048 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1696250082161.5bf5964d15ce023a31881482c13240a2. 2023-10-02T12:35:55,049 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing ee1a09a86970bc5e288b8a3080cb4647, disabling compactions & flushes 2023-10-02T12:35:55,049 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. 2023-10-02T12:35:55,049 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. 2023-10-02T12:35:55,049 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. after waiting 0 ms 2023-10-02T12:35:55,049 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. 2023-10-02T12:35:55,053 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/data/ns4/test-16962500765863/ee1a09a86970bc5e288b8a3080cb4647/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:35:55,053 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:35:55,054 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. 2023-10-02T12:35:55,054 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for ee1a09a86970bc5e288b8a3080cb4647: 2023-10-02T12:35:55,054 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16962500765863,,1696250080824.ee1a09a86970bc5e288b8a3080cb4647. 2023-10-02T12:35:55,214 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,40211,1696250065770; all regions closed. 2023-10-02T12:35:55,222 DEBUG [RS:0;jenkins-hbase3:40211 {}] wal.AbstractFSWAL(1062): Moved 2 WAL file(s) to /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs 2023-10-02T12:35:55,222 INFO [RS:0;jenkins-hbase3:40211 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770.meta:.meta(num 1696250142332) 2023-10-02T12:35:55,232 DEBUG [RS:0;jenkins-hbase3:40211 {}] wal.AbstractFSWAL(1062): Moved 7 WAL file(s) to /user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/oldWALs 2023-10-02T12:35:55,232 INFO [RS:0;jenkins-hbase3:40211 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C40211%2C1696250065770:(num 1696250142314) 2023-10-02T12:35:55,232 DEBUG [RS:0;jenkins-hbase3:40211 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:55,232 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:35:55,232 INFO [RS:0;jenkins-hbase3:40211 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-10-02T12:35:55,233 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:35:55,234 INFO [RS:0;jenkins-hbase3:40211 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:40211 2023-10-02T12:35:55,242 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/jenkins-hbase3.apache.org,40211,1696250065770 2023-10-02T12:35:55,242 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-10-02T12:35:55,245 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,40211,1696250065770] 2023-10-02T12:35:55,246 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,40211,1696250065770; numProcessing=1 2023-10-02T12:35:55,247 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /1/draining/jenkins-hbase3.apache.org,40211,1696250065770 already deleted, retry=false 2023-10-02T12:35:55,247 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,40211,1696250065770 expired; onlineServers=0 2023-10-02T12:35:55,247 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,39919,1696250062021' ***** 2023-10-02T12:35:55,247 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-10-02T12:35:55,248 DEBUG [M:0;jenkins-hbase3:39919 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68adcb8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:35:55,248 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:35:55,255 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2023-10-02T12:35:55,255 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:35:55,256 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:35:55,293 INFO [M:0;jenkins-hbase3:39919 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@673fe3{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:35:55,294 INFO [M:0;jenkins-hbase3:39919 {}] server.AbstractConnector(383): Stopped ServerConnector@3ecfb4c5{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:35:55,294 INFO [M:0;jenkins-hbase3:39919 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:35:55,295 INFO [M:0;jenkins-hbase3:39919 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@777c649c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:35:55,295 INFO [M:0;jenkins-hbase3:39919 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1c0a99ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/hadoop.log.dir/,STOPPED} 2023-10-02T12:35:55,296 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,39919,1696250062021 2023-10-02T12:35:55,296 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,39919,1696250062021; all regions closed. 2023-10-02T12:35:55,296 DEBUG [M:0;jenkins-hbase3:39919 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:55,296 INFO [M:0;jenkins-hbase3:39919 {}] master.HMaster(1665): Stopping master jetty server 2023-10-02T12:35:55,296 INFO [M:0;jenkins-hbase3:39919 {}] server.AbstractConnector(383): Stopped ServerConnector@1fdf5b65{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:35:55,297 DEBUG [M:0;jenkins-hbase3:39919 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-10-02T12:35:55,297 INFO [M:0;jenkins-hbase3:39919 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-10-02T12:35:55,297 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-10-02T12:35:55,297 DEBUG [M:0;jenkins-hbase3:39919 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1991d3c4 to 127.0.0.1:64600 2023-10-02T12:35:55,297 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696250067842 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696250067842,5,FailOnTimeoutGroup] 2023-10-02T12:35:55,297 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696250067842 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696250067842,5,FailOnTimeoutGroup] 2023-10-02T12:35:55,297 DEBUG [M:0;jenkins-hbase3:39919 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:35:55,297 DEBUG [M:0;jenkins-hbase3:39919 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-10-02T12:35:55,297 INFO [M:0;jenkins-hbase3:39919 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-10-02T12:35:55,297 DEBUG [M:0;jenkins-hbase3:39919 {}] master.HMaster(1688): Stopping service threads 2023-10-02T12:35:55,298 INFO [M:0;jenkins-hbase3:39919 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-10-02T12:35:55,298 ERROR [M:0;jenkins-hbase3:39919 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-10-02T12:35:55,298 INFO [M:0;jenkins-hbase3:39919 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-10-02T12:35:55,298 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-10-02T12:35:55,299 DEBUG [M:0;jenkins-hbase3:39919 {}] zookeeper.ZKUtil(347): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2023-10-02T12:35:55,299 WARN [M:0;jenkins-hbase3:39919 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-10-02T12:35:55,299 INFO [M:0;jenkins-hbase3:39919 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-10-02T12:35:55,299 INFO [M:0;jenkins-hbase3:39919 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-10-02T12:35:55,300 DEBUG [M:0;jenkins-hbase3:39919 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:35:55,311 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:35:55,311 DEBUG [M:0;jenkins-hbase3:39919 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:35:55,311 DEBUG [M:0;jenkins-hbase3:39919 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:35:55,311 DEBUG [M:0;jenkins-hbase3:39919 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:35:55,311 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=201.89 KB heapSize=244.62 KB 2023-10-02T12:35:55,323 DEBUG [M:0;jenkins-hbase3:39919 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a76acec65f5842d99210d9947633cf56 is 88, key is hbase:meta,,1/info:sn/1696250068872/Put/seqid=0 2023-10-02T12:35:55,327 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a76acec65f5842d99210d9947633cf56 2023-10-02T12:35:55,345 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-02T12:35:55,345 INFO [RS:0;jenkins-hbase3:40211 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,40211,1696250065770; zookeeper connection closed. 2023-10-02T12:35:55,345 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40211-0x101732b40f20001, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-02T12:35:55,345 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@66438305 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@66438305 2023-10-02T12:35:55,345 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-10-02T12:35:55,350 DEBUG [M:0;jenkins-hbase3:39919 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/48b76117099843859398206417f29ee7 is 987, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1696250083700/Put/seqid=0 2023-10-02T12:35:55,755 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=201.30 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/48b76117099843859398206417f29ee7 2023-10-02T12:35:55,760 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 48b76117099843859398206417f29ee7 2023-10-02T12:35:55,773 DEBUG [M:0;jenkins-hbase3:39919 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/01c2e76431f34332860fd213238c9d04 is 82, key is jenkins-hbase3.apache.org,40211,1696250065770/rs:state/1696250067870/Put/seqid=0 2023-10-02T12:35:56,180 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/01c2e76431f34332860fd213238c9d04 2023-10-02T12:35:56,187 DEBUG [M:0;jenkins-hbase3:39919 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a76acec65f5842d99210d9947633cf56 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a76acec65f5842d99210d9947633cf56 2023-10-02T12:35:56,192 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a76acec65f5842d99210d9947633cf56, entries=8, sequenceid=505, filesize=5.5 K 2023-10-02T12:35:56,192 DEBUG [M:0;jenkins-hbase3:39919 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/48b76117099843859398206417f29ee7 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/48b76117099843859398206417f29ee7 2023-10-02T12:35:56,197 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 48b76117099843859398206417f29ee7 2023-10-02T12:35:56,197 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/48b76117099843859398206417f29ee7, entries=53, sequenceid=505, filesize=16.5 K 2023-10-02T12:35:56,197 DEBUG [M:0;jenkins-hbase3:39919 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/01c2e76431f34332860fd213238c9d04 as hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/01c2e76431f34332860fd213238c9d04 2023-10-02T12:35:56,202 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:33727/user/jenkins/test-data/7f9da5dd-190f-f956-cd3c-fa62ee774fc3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/01c2e76431f34332860fd213238c9d04, entries=1, sequenceid=505, filesize=5.1 K 2023-10-02T12:35:56,202 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.HRegion(3022): Finished flush of dataSize ~201.89 KB/206735, heapSize ~244.32 KB/250184, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 891ms, sequenceid=505, compaction requested=false 2023-10-02T12:35:56,207 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:35:56,207 DEBUG [M:0;jenkins-hbase3:39919 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:35:56,209 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:35:56,209 INFO [M:0;jenkins-hbase3:39919 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-10-02T12:35:56,210 INFO [M:0;jenkins-hbase3:39919 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-10-02T12:35:56,210 INFO [M:0;jenkins-hbase3:39919 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:39919 2023-10-02T12:35:56,212 DEBUG [M:0;jenkins-hbase3:39919 {}] zookeeper.RecoverableZooKeeper(224): Node /1/rs/jenkins-hbase3.apache.org,39919,1696250062021 already deleted, retry=false 2023-10-02T12:35:56,315 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-02T12:35:56,315 INFO [M:0;jenkins-hbase3:39919 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,39919,1696250062021; zookeeper connection closed. 2023-10-02T12:35:56,315 DEBUG [Listener at localhost/34813-EventThread {}] zookeeper.ZKWatcher(604): master:39919-0x101732b40f20000, quorum=127.0.0.1:64600, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-10-02T12:35:56,322 WARN [Listener at jenkins-hbase3.apache.org/34739 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-10-02T12:35:56,330 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:35:56,440 WARN [BP-1578325291-172.31.12.81-1696250058286 heartbeating to localhost/127.0.0.1:33727 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-10-02T12:35:56,440 WARN [BP-1578325291-172.31.12.81-1696250058286 heartbeating to localhost/127.0.0.1:33727 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1578325291-172.31.12.81-1696250058286 (Datanode Uuid 0f2caf88-88c6-4ff7-a391-75aaacc42dcb) service to localhost/127.0.0.1:33727 2023-10-02T12:35:56,442 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/cluster_9dfd02cb-0bbd-743a-fb9e-a562fae510de/dfs/data/data1/current/BP-1578325291-172.31.12.81-1696250058286 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:35:56,443 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c64e9774-2b4d-1f84-e1e3-27112c8d7e07/cluster_9dfd02cb-0bbd-743a-fb9e-a562fae510de/dfs/data/data2/current/BP-1578325291-172.31.12.81-1696250058286 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:35:56,477 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:35:56,605 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2023-10-02T12:35:56,698 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-10-02T12:35:56,698 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-10-02T12:35:56,709 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:35:56,826 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:35:56,936 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:35:56,936 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:35:57,042 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-10-02T12:35:57,049 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-10-02T12:35:57,051 ERROR [Thread[Thread-286,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:35:57,060 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:35:57,163 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:35:57,163 INFO [Listener at jenkins-hbase3.apache.org/34739 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped