2023-11-12 09:31:24,097 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@229f66ed 2023-11-12 09:31:24,114 main DEBUG Took 0.014309 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-11-12 09:31:24,115 main DEBUG PluginManager 'Core' found 129 plugins 2023-11-12 09:31:24,115 main DEBUG PluginManager 'Level' found 0 plugins 2023-11-12 09:31:24,116 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-11-12 09:31:24,118 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:31:24,128 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-11-12 09:31:24,143 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:31:24,146 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,147 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:31:24,147 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:31:24,149 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,149 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:31:24,150 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:31:24,151 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:31:24,152 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,152 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:31:24,153 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:31:24,154 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,154 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:31:24,154 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,154 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:31:24,155 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,155 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:31:24,156 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,156 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:31:24,156 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,157 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-11-12 09:31:24,159 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:31:24,162 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-11-12 09:31:24,164 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-11-12 09:31:24,165 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-11-12 09:31:24,166 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-11-12 09:31:24,166 main DEBUG PluginManager 'Converter' found 47 plugins 2023-11-12 09:31:24,179 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-11-12 09:31:24,182 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-11-12 09:31:24,183 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-11-12 09:31:24,183 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-11-12 09:31:24,184 main DEBUG createAppenders(={Console}) 2023-11-12 09:31:24,184 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@229f66ed initialized 2023-11-12 09:31:24,185 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@229f66ed 2023-11-12 09:31:24,185 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@229f66ed OK. 2023-11-12 09:31:24,185 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-11-12 09:31:24,186 main DEBUG OutputStream closed 2023-11-12 09:31:24,186 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-11-12 09:31:24,186 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-11-12 09:31:24,186 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@498d318c OK 2023-11-12 09:31:24,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-11-12 09:31:24,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-11-12 09:31:24,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-11-12 09:31:24,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-11-12 09:31:24,237 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-11-12 09:31:24,238 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-11-12 09:31:24,238 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-11-12 09:31:24,238 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-11-12 09:31:24,238 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-11-12 09:31:24,239 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-11-12 09:31:24,239 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-11-12 09:31:24,239 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-11-12 09:31:24,240 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-11-12 09:31:24,240 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-11-12 09:31:24,240 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-11-12 09:31:24,240 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-11-12 09:31:24,241 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-11-12 09:31:24,242 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-11-12 09:31:24,245 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-12 09:31:24,245 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4d5b6aac) with optional ClassLoader: null 2023-11-12 09:31:24,245 main DEBUG Shutdown hook enabled. Registering a new one. 2023-11-12 09:31:24,246 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@4d5b6aac] started OK. 2023-11-12T09:31:24,280 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupMultipleDeletes timeout: 13 mins 2023-11-12 09:31:24,285 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-11-12 09:31:24,285 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-12T09:31:24,646 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b 2023-11-12T09:31:24,652 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-12T09:31:24,656 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:31:24,657 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-11-12T09:31:24,661 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/cluster_7b0087cc-f951-95b8-513e-bf398edab43a, deleteOnExit=true 2023-11-12T09:31:24,662 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-11-12T09:31:24,662 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/test.cache.data in system properties and HBase conf 2023-11-12T09:31:24,663 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop.tmp.dir in system properties and HBase conf 2023-11-12T09:31:24,663 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop.log.dir in system properties and HBase conf 2023-11-12T09:31:24,664 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-12T09:31:24,664 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-12T09:31:24,664 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-12T09:31:24,799 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-11-12T09:31:25,321 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-11-12T09:31:25,326 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:31:25,326 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:31:25,326 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-12T09:31:25,327 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:31:25,327 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-12T09:31:25,327 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-12T09:31:25,327 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:31:25,328 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:31:25,328 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-12T09:31:25,328 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/nfs.dump.dir in system properties and HBase conf 2023-11-12T09:31:25,329 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/java.io.tmpdir in system properties and HBase conf 2023-11-12T09:31:25,329 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:31:25,329 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-12T09:31:25,329 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-11-12T09:31:25,833 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:31:25,838 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:31:26,146 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-11-12T09:31:26,312 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@22eaf849 via org.mortbay.log.Slf4jLog 2023-11-12T09:31:26,335 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:31:26,387 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:31:26,428 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/java.io.tmpdir/Jetty_localhost_36095_hdfs____pkfj5f/webapp 2023-11-12T09:31:26,589 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36095 2023-11-12T09:31:26,614 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:31:26,614 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:31:27,220 WARN [Listener at localhost/38513 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:31:27,287 WARN [Listener at localhost/38513 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-12T09:31:27,309 WARN [Listener at localhost/38513 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:31:27,318 INFO [Listener at localhost/38513 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:31:27,325 INFO [Listener at localhost/38513 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/java.io.tmpdir/Jetty_localhost_41527_datanode____7mygas/webapp 2023-11-12T09:31:27,423 INFO [Listener at localhost/38513 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41527 2023-11-12T09:31:27,697 WARN [Listener at localhost/32839 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:31:27,997 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-12T09:31:28,152 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x8891479ffe929d51: Processing first storage report for DS-d9986012-1171-4bc5-83f5-e236c8649a49 from datanode 3750ebf4-1eec-4283-b25d-5998f5bf7e75 2023-11-12T09:31:28,154 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x8891479ffe929d51: from storage DS-d9986012-1171-4bc5-83f5-e236c8649a49 node DatanodeRegistration(127.0.0.1:43915, datanodeUuid=3750ebf4-1eec-4283-b25d-5998f5bf7e75, infoPort=43895, infoSecurePort=0, ipcPort=32839, storageInfo=lv=-57;cid=testClusterID;nsid=56049448;c=1699781485924), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-11-12T09:31:28,154 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x8891479ffe929d51: Processing first storage report for DS-2fcfa7d2-9d15-4a92-8e90-193a653771e6 from datanode 3750ebf4-1eec-4283-b25d-5998f5bf7e75 2023-11-12T09:31:28,154 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x8891479ffe929d51: from storage DS-2fcfa7d2-9d15-4a92-8e90-193a653771e6 node DatanodeRegistration(127.0.0.1:43915, datanodeUuid=3750ebf4-1eec-4283-b25d-5998f5bf7e75, infoPort=43895, infoSecurePort=0, ipcPort=32839, storageInfo=lv=-57;cid=testClusterID;nsid=56049448;c=1699781485924), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-12T09:31:28,242 DEBUG [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b 2023-11-12T09:31:28,412 INFO [Listener at localhost/32839 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/cluster_7b0087cc-f951-95b8-513e-bf398edab43a/zookeeper_0, clientPort=55425, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/cluster_7b0087cc-f951-95b8-513e-bf398edab43a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/cluster_7b0087cc-f951-95b8-513e-bf398edab43a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2023-11-12T09:31:28,433 INFO [Listener at localhost/32839 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=55425 2023-11-12T09:31:28,441 INFO [Listener at localhost/32839 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:31:28,443 INFO [Listener at localhost/32839 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:31:29,119 INFO [Listener at localhost/32839 {}] util.FSUtils(462): Created version file at hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f with version=8 2023-11-12T09:31:29,119 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/hbase-staging 2023-11-12T09:31:29,136 DEBUG [Listener at localhost/32839 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-11-12T09:31:29,137 DEBUG [Listener at localhost/32839 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-11-12T09:31:29,137 DEBUG [Listener at localhost/32839 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-11-12T09:31:29,137 DEBUG [Listener at localhost/32839 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-11-12T09:31:29,361 DEBUG [Listener at localhost/32839 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-11-12T09:31:30,155 INFO [Listener at localhost/32839 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-11-12T09:31:30,225 INFO [Listener at localhost/32839 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:31:30,226 INFO [Listener at localhost/32839 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-12T09:31:30,226 INFO [Listener at localhost/32839 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-12T09:31:30,226 INFO [Listener at localhost/32839 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:31:30,227 INFO [Listener at localhost/32839 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-12T09:31:30,436 INFO [Listener at localhost/32839 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-12T09:31:30,569 INFO [Listener at localhost/32839 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-11-12T09:31:30,589 DEBUG [Listener at localhost/32839 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-11-12T09:31:30,595 INFO [Listener at localhost/32839 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-12T09:31:30,644 DEBUG [Listener at localhost/32839 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 31581 (auto-detected) 2023-11-12T09:31:30,646 DEBUG [Listener at localhost/32839 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:0f:ff:fe:94:aa:bf (auto-detected) 2023-11-12T09:31:30,683 INFO [Listener at localhost/32839 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:35261 2023-11-12T09:31:30,698 INFO [Listener at localhost/32839 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:31:30,700 INFO [Listener at localhost/32839 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:31:30,722 INFO [Listener at localhost/32839 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:35261 connecting to ZooKeeper ensemble=127.0.0.1:55425 2023-11-12T09:31:30,784 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:352610x0, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-12T09:31:30,789 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:35261-0x1004cf91cf70000 connected 2023-11-12T09:31:30,831 DEBUG [Listener at localhost/32839 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-12T09:31:30,835 DEBUG [Listener at localhost/32839 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:31:30,839 DEBUG [Listener at localhost/32839 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-12T09:31:30,852 DEBUG [Listener at localhost/32839 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35261 2023-11-12T09:31:30,852 DEBUG [Listener at localhost/32839 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35261 2023-11-12T09:31:30,853 DEBUG [Listener at localhost/32839 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35261 2023-11-12T09:31:30,858 DEBUG [Listener at localhost/32839 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35261 2023-11-12T09:31:30,858 DEBUG [Listener at localhost/32839 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35261 2023-11-12T09:31:30,922 INFO [Listener at localhost/32839 {}] log.Log(170): Logging initialized @7610ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-11-12T09:31:31,062 INFO [Listener at localhost/32839 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-12T09:31:31,063 INFO [Listener at localhost/32839 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-12T09:31:31,064 INFO [Listener at localhost/32839 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-12T09:31:31,066 INFO [Listener at localhost/32839 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-11-12T09:31:31,067 INFO [Listener at localhost/32839 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-12T09:31:31,067 INFO [Listener at localhost/32839 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-12T09:31:31,071 INFO [Listener at localhost/32839 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-12T09:31:31,228 INFO [Listener at localhost/32839 {}] http.HttpServer(1227): Jetty bound to port 44999 2023-11-12T09:31:31,230 INFO [Listener at localhost/32839 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:31:31,271 INFO [Listener at localhost/32839 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:31:31,274 INFO [Listener at localhost/32839 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@253fe48{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop.log.dir/,AVAILABLE} 2023-11-12T09:31:31,275 INFO [Listener at localhost/32839 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:31:31,276 INFO [Listener at localhost/32839 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@143113d4{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-12T09:31:31,353 INFO [Listener at localhost/32839 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-12T09:31:31,371 INFO [Listener at localhost/32839 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-12T09:31:31,372 INFO [Listener at localhost/32839 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-12T09:31:31,375 INFO [Listener at localhost/32839 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-11-12T09:31:31,386 INFO [Listener at localhost/32839 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:31:32,698 INFO [Listener at localhost/32839 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@6795993c{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-12T09:31:32,710 INFO [Listener at localhost/32839 {}] server.AbstractConnector(333): Started ServerConnector@99139ca{HTTP/1.1, (http/1.1)}{0.0.0.0:44999} 2023-11-12T09:31:32,710 INFO [Listener at localhost/32839 {}] server.Server(415): Started @9399ms 2023-11-12T09:31:32,714 INFO [Listener at localhost/32839 {}] master.HMaster(486): hbase.rootdir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f, hbase.cluster.distributed=false 2023-11-12T09:31:32,777 INFO [Listener at localhost/32839 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-11-12T09:31:32,777 INFO [Listener at localhost/32839 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:31:32,777 INFO [Listener at localhost/32839 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-12T09:31:32,777 INFO [Listener at localhost/32839 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-12T09:31:32,777 INFO [Listener at localhost/32839 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:31:32,777 INFO [Listener at localhost/32839 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-12T09:31:32,786 INFO [Listener at localhost/32839 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-12T09:31:32,788 INFO [Listener at localhost/32839 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-12T09:31:32,789 INFO [Listener at localhost/32839 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:37711 2023-11-12T09:31:32,792 INFO [Listener at localhost/32839 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-11-12T09:31:32,800 DEBUG [Listener at localhost/32839 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-11-12T09:31:32,801 INFO [Listener at localhost/32839 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:31:32,804 INFO [Listener at localhost/32839 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:31:32,807 INFO [Listener at localhost/32839 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:37711 connecting to ZooKeeper ensemble=127.0.0.1:55425 2023-11-12T09:31:32,811 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:377110x0, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-12T09:31:32,812 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:37711-0x1004cf91cf70001 connected 2023-11-12T09:31:32,813 DEBUG [Listener at localhost/32839 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-12T09:31:32,815 DEBUG [Listener at localhost/32839 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:31:32,816 DEBUG [Listener at localhost/32839 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-12T09:31:32,817 DEBUG [Listener at localhost/32839 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37711 2023-11-12T09:31:32,817 DEBUG [Listener at localhost/32839 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37711 2023-11-12T09:31:32,818 DEBUG [Listener at localhost/32839 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37711 2023-11-12T09:31:32,818 DEBUG [Listener at localhost/32839 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37711 2023-11-12T09:31:32,819 DEBUG [Listener at localhost/32839 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37711 2023-11-12T09:31:32,822 INFO [Listener at localhost/32839 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-12T09:31:32,823 INFO [Listener at localhost/32839 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-12T09:31:32,823 INFO [Listener at localhost/32839 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-12T09:31:32,824 INFO [Listener at localhost/32839 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-11-12T09:31:32,824 INFO [Listener at localhost/32839 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-12T09:31:32,824 INFO [Listener at localhost/32839 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-12T09:31:32,824 INFO [Listener at localhost/32839 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-12T09:31:32,826 INFO [Listener at localhost/32839 {}] http.HttpServer(1227): Jetty bound to port 36059 2023-11-12T09:31:32,827 INFO [Listener at localhost/32839 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:31:32,829 INFO [Listener at localhost/32839 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:31:32,829 INFO [Listener at localhost/32839 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5ffd5097{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop.log.dir/,AVAILABLE} 2023-11-12T09:31:32,830 INFO [Listener at localhost/32839 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:31:32,830 INFO [Listener at localhost/32839 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@26457e15{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-12T09:31:32,843 INFO [Listener at localhost/32839 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-12T09:31:32,845 INFO [Listener at localhost/32839 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-12T09:31:32,845 INFO [Listener at localhost/32839 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-12T09:31:32,845 INFO [Listener at localhost/32839 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-11-12T09:31:32,847 INFO [Listener at localhost/32839 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:31:32,853 INFO [Listener at localhost/32839 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@267f8c07{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-12T09:31:32,854 INFO [Listener at localhost/32839 {}] server.AbstractConnector(333): Started ServerConnector@24fa6f21{HTTP/1.1, (http/1.1)}{0.0.0.0:36059} 2023-11-12T09:31:32,854 INFO [Listener at localhost/32839 {}] server.Server(415): Started @9542ms 2023-11-12T09:31:32,860 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:31:32,890 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@7843ecc9{HTTP/1.1, (http/1.1)}{0.0.0.0:43225} 2023-11-12T09:31:32,890 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @9578ms 2023-11-12T09:31:32,890 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2389): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,35261,1699781489344 2023-11-12T09:31:32,906 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:31:32,906 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:31:32,909 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,35261,1699781489344 2023-11-12T09:31:32,918 DEBUG [M:0;jenkins-hbase3:35261 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:35261 2023-11-12T09:31:32,930 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-12T09:31:32,930 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-12T09:31:32,930 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:32,930 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:32,931 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-12T09:31:32,933 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,35261,1699781489344 from backup master directory 2023-11-12T09:31:32,933 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-12T09:31:32,936 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,35261,1699781489344 2023-11-12T09:31:32,937 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:31:32,937 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:31:32,938 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-12T09:31:32,938 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,35261,1699781489344 2023-11-12T09:31:32,941 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-11-12T09:31:32,944 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-11-12T09:31:33,026 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/hbase.id with ID: bca8bb0a-1d91-469d-89ac-ff5e779b77de 2023-11-12T09:31:33,067 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:31:33,083 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:33,083 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:33,125 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:31:33,127 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-11-12T09:31:33,146 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-11-12T09:31:33,147 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-11-12T09:31:33,148 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:31:33,158 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:31:33,160 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:31:33,234 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store 2023-11-12T09:31:33,259 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-11-12T09:31:33,259 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:33,260 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-12T09:31:33,260 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:31:33,260 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:31:33,260 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-12T09:31:33,260 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:31:33,261 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:31:33,261 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:31:33,263 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/.initializing 2023-11-12T09:31:33,263 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/WALs/jenkins-hbase3.apache.org,35261,1699781489344 2023-11-12T09:31:33,273 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-12T09:31:33,289 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C35261%2C1699781489344, suffix=, logDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/WALs/jenkins-hbase3.apache.org,35261,1699781489344, archiveDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/oldWALs, maxLogs=10 2023-11-12T09:31:33,329 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/WALs/jenkins-hbase3.apache.org,35261,1699781489344/jenkins-hbase3.apache.org%2C35261%2C1699781489344.1699781493297, exclude list is [], retry=0 2023-11-12T09:31:33,364 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:31:33,372 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:499) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:31:33,451 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/WALs/jenkins-hbase3.apache.org,35261,1699781489344/jenkins-hbase3.apache.org%2C35261%2C1699781489344.1699781493297 2023-11-12T09:31:33,452 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:31:33,453 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:31:33,453 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:33,458 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:31:33,459 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:31:33,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:31:33,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-11-12T09:31:33,571 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:33,574 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:31:33,575 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:31:33,578 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-11-12T09:31:33,578 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:33,579 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:31:33,580 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:31:33,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-11-12T09:31:33,582 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:33,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:31:33,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:31:33,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-11-12T09:31:33,586 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:33,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:31:33,593 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:31:33,594 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:31:33,610 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-11-12T09:31:33,614 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:31:33,619 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:31:33,620 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10436831200, jitterRate=-0.027994349598884583}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-11-12T09:31:33,625 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:31:33,626 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-11-12T09:31:33,666 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38c0d22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:31:33,721 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(861): No meta location available on zookeeper, skip migrating... 2023-11-12T09:31:33,742 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-11-12T09:31:33,742 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-11-12T09:31:33,745 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-11-12T09:31:33,747 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-11-12T09:31:33,753 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 6 msec 2023-11-12T09:31:33,753 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-11-12T09:31:33,789 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-11-12T09:31:33,806 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-11-12T09:31:33,809 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-11-12T09:31:33,812 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-11-12T09:31:33,813 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-11-12T09:31:33,815 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-11-12T09:31:33,818 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-11-12T09:31:33,825 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-11-12T09:31:33,828 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-11-12T09:31:33,829 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-11-12T09:31:33,830 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-11-12T09:31:33,846 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-11-12T09:31:33,848 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-11-12T09:31:33,853 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-12T09:31:33,853 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-12T09:31:33,854 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:33,854 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:33,854 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(805): Active/primary master=jenkins-hbase3.apache.org,35261,1699781489344, sessionid=0x1004cf91cf70000, setting cluster-up flag (Was=false) 2023-11-12T09:31:33,864 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-11-12T09:31:33,875 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:33,875 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:33,882 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-11-12T09:31:33,883 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,35261,1699781489344 2023-11-12T09:31:33,887 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:33,887 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:33,894 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-11-12T09:31:33,895 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,35261,1699781489344 2023-11-12T09:31:33,900 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:33,900 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:33,907 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-11-12T09:31:33,909 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,35261,1699781489344 2023-11-12T09:31:33,970 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:37711 2023-11-12T09:31:33,972 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(997): ClusterId : bca8bb0a-1d91-469d-89ac-ff5e779b77de 2023-11-12T09:31:33,974 INFO [RS:0;jenkins-hbase3:37711 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-11-12T09:31:33,977 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-11-12T09:31:33,986 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-11-12T09:31:33,986 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-11-12T09:31:33,990 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-11-12T09:31:33,990 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-11-12T09:31:33,994 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-11-12T09:31:33,996 DEBUG [RS:0;jenkins-hbase3:37711 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e61aea0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:31:33,999 DEBUG [RS:0;jenkins-hbase3:37711 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6a19e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-11-12T09:31:34,004 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-11-12T09:31:34,005 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-11-12T09:31:34,005 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-11-12T09:31:34,009 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,35261,1699781489344 with isa=jenkins-hbase3.apache.org/172.31.12.81:37711, startcode=1699781492776 2023-11-12T09:31:34,033 DEBUG [RS:0;jenkins-hbase3:37711 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-12T09:31:34,085 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-11-12T09:31:34,102 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-11-12T09:31:34,108 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-11-12T09:31:34,119 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,35261,1699781489344 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-11-12T09:31:34,125 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:31:34,125 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:31:34,125 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:31:34,125 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:31:34,125 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-11-12T09:31:34,126 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:31:34,126 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-11-12T09:31:34,126 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:31:34,127 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:43213, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-12T09:31:34,142 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1699781524142 2023-11-12T09:31:34,145 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-11-12T09:31:34,147 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35261 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3212) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:592) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-12T09:31:34,150 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-11-12T09:31:34,155 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-11-12T09:31:34,156 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-11-12T09:31:34,163 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:34,163 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-12T09:31:34,164 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-11-12T09:31:34,165 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-11-12T09:31:34,166 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-11-12T09:31:34,167 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-11-12T09:31:34,167 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-11-12T09:31:34,170 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:34,172 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-11-12T09:31:34,175 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-11-12T09:31:34,176 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-11-12T09:31:34,194 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x341b23bc to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:31:34,198 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-11-12T09:31:34,198 WARN [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-11-12T09:31:34,203 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a138fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:31:34,203 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-12T09:31:34,206 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-11-12T09:31:34,207 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-11-12T09:31:34,210 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781494208,5,FailOnTimeoutGroup] 2023-11-12T09:31:34,210 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781494210,5,FailOnTimeoutGroup] 2023-11-12T09:31:34,210 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:34,210 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1636): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-11-12T09:31:34,211 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:34,212 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:34,300 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,35261,1699781489344 with isa=jenkins-hbase3.apache.org/172.31.12.81:37711, startcode=1699781492776 2023-11-12T09:31:34,305 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35261 {}] master.ServerManager(396): Registering regionserver=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:34,317 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f 2023-11-12T09:31:34,317 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:38513 2023-11-12T09:31:34,317 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=44999 2023-11-12T09:31:34,328 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-11-12T09:31:34,328 DEBUG [RS:0;jenkins-hbase3:37711 {}] zookeeper.ZKUtil(111): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:34,329 WARN [RS:0;jenkins-hbase3:37711 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-12T09:31:34,329 INFO [RS:0;jenkins-hbase3:37711 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:31:34,329 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:34,332 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,37711,1699781492776] 2023-11-12T09:31:34,346 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-11-12T09:31:34,362 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-11-12T09:31:34,377 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-11-12T09:31:34,382 INFO [RS:0;jenkins-hbase3:37711 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-11-12T09:31:34,382 INFO [RS:0;jenkins-hbase3:37711 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:34,383 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-11-12T09:31:34,393 INFO [RS:0;jenkins-hbase3:37711 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:34,393 DEBUG [RS:0;jenkins-hbase3:37711 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:31:34,393 DEBUG [RS:0;jenkins-hbase3:37711 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:31:34,394 DEBUG [RS:0;jenkins-hbase3:37711 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:31:34,394 DEBUG [RS:0;jenkins-hbase3:37711 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:31:34,394 DEBUG [RS:0;jenkins-hbase3:37711 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:31:34,394 DEBUG [RS:0;jenkins-hbase3:37711 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-11-12T09:31:34,394 DEBUG [RS:0;jenkins-hbase3:37711 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:31:34,394 DEBUG [RS:0;jenkins-hbase3:37711 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:31:34,394 DEBUG [RS:0;jenkins-hbase3:37711 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:31:34,394 DEBUG [RS:0;jenkins-hbase3:37711 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:31:34,395 DEBUG [RS:0;jenkins-hbase3:37711 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:31:34,395 DEBUG [RS:0;jenkins-hbase3:37711 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-11-12T09:31:34,395 DEBUG [RS:0;jenkins-hbase3:37711 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-11-12T09:31:34,396 INFO [RS:0;jenkins-hbase3:37711 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:34,396 INFO [RS:0;jenkins-hbase3:37711 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:34,396 INFO [RS:0;jenkins-hbase3:37711 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:34,396 INFO [RS:0;jenkins-hbase3:37711 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:34,396 INFO [RS:0;jenkins-hbase3:37711 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37711,1699781492776-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-12T09:31:34,413 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-11-12T09:31:34,416 INFO [RS:0;jenkins-hbase3:37711 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37711,1699781492776-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:34,433 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,37711,1699781492776 started 2023-11-12T09:31:34,433 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,37711,1699781492776, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:37711, sessionid=0x1004cf91cf70001 2023-11-12T09:31:34,434 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-11-12T09:31:34,434 DEBUG [RS:0;jenkins-hbase3:37711 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:34,434 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37711,1699781492776' 2023-11-12T09:31:34,434 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-11-12T09:31:34,435 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-11-12T09:31:34,436 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-11-12T09:31:34,436 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-11-12T09:31:34,436 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37711,1699781492776' 2023-11-12T09:31:34,436 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:31:34,437 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:31:34,437 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-11-12T09:31:34,437 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-11-12T09:31:34,438 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-11-12T09:31:34,438 DEBUG [RS:0;jenkins-hbase3:37711 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:34,438 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37711,1699781492776' 2023-11-12T09:31:34,438 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-11-12T09:31:34,438 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-11-12T09:31:34,439 DEBUG [RS:0;jenkins-hbase3:37711 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-11-12T09:31:34,439 INFO [RS:0;jenkins-hbase3:37711 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-11-12T09:31:34,439 INFO [RS:0;jenkins-hbase3:37711 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-11-12T09:31:34,550 INFO [RS:0;jenkins-hbase3:37711 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-12T09:31:34,553 INFO [RS:0;jenkins-hbase3:37711 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37711%2C1699781492776, suffix=, logDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776, archiveDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs, maxLogs=32 2023-11-12T09:31:34,565 DEBUG [RS:0;jenkins-hbase3:37711 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781494554, exclude list is [], retry=0 2023-11-12T09:31:34,571 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:31:34,576 INFO [RS:0;jenkins-hbase3:37711 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781494554 2023-11-12T09:31:34,576 DEBUG [RS:0;jenkins-hbase3:37711 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:31:34,593 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-11-12T09:31:34,594 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f 2023-11-12T09:31:35,011 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:35,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-12T09:31:35,016 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-12T09:31:35,016 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:35,017 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:31:35,017 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-12T09:31:35,019 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-12T09:31:35,019 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:35,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:31:35,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-12T09:31:35,022 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-12T09:31:35,023 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:35,024 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:31:35,028 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740 2023-11-12T09:31:35,029 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740 2023-11-12T09:31:35,031 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-12T09:31:35,033 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-12T09:31:35,037 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:31:35,038 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10790538560, jitterRate=0.00494721531867981}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-12T09:31:35,039 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-12T09:31:35,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-12T09:31:35,040 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-12T09:31:35,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-12T09:31:35,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-12T09:31:35,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-12T09:31:35,041 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-12T09:31:35,041 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-12T09:31:35,044 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-11-12T09:31:35,045 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-11-12T09:31:35,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-11-12T09:31:35,066 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-11-12T09:31:35,070 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-11-12T09:31:35,223 DEBUG [jenkins-hbase3:35261 {}] assignment.AssignmentManager(2319): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-11-12T09:31:35,234 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:35,240 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,37711,1699781492776, state=OPENING 2023-11-12T09:31:35,251 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-11-12T09:31:35,253 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:35,253 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:35,254 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:31:35,254 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:31:35,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,37711,1699781492776}] 2023-11-12T09:31:35,592 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:35,595 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-11-12T09:31:35,598 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38556, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-11-12T09:31:35,611 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-11-12T09:31:35,611 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:31:35,612 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-11-12T09:31:35,615 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta, suffix=.meta, logDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776, archiveDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs, maxLogs=32 2023-11-12T09:31:35,628 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781495617.meta, exclude list is [], retry=0 2023-11-12T09:31:35,632 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:31:35,636 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781495617.meta 2023-11-12T09:31:35,636 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:31:35,637 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:31:35,642 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:31:35,643 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-11-12T09:31:35,660 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-11-12T09:31:35,660 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-11-12T09:31:35,665 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-11-12T09:31:35,665 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:35,666 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-11-12T09:31:35,666 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-11-12T09:31:35,669 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-12T09:31:35,670 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-12T09:31:35,670 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:35,671 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:31:35,671 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-12T09:31:35,672 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-12T09:31:35,673 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:35,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:31:35,674 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-12T09:31:35,675 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-12T09:31:35,675 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:35,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:31:35,677 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740 2023-11-12T09:31:35,680 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740 2023-11-12T09:31:35,682 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-12T09:31:35,684 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-12T09:31:35,686 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11967517760, jitterRate=0.11456194519996643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-12T09:31:35,687 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-12T09:31:35,699 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1699781495583 2023-11-12T09:31:35,720 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:35,722 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,37711,1699781492776, state=OPEN 2023-11-12T09:31:35,724 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-11-12T09:31:35,725 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-11-12T09:31:35,725 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-12T09:31:35,725 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-12T09:31:35,725 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:31:35,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:31:35,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-11-12T09:31:35,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,37711,1699781492776 in 467 msec 2023-11-12T09:31:35,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-11-12T09:31:35,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 678 msec 2023-11-12T09:31:35,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.7580 sec 2023-11-12T09:31:35,748 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1064): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1699781495748, completionTime=-1 2023-11-12T09:31:35,748 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(816): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-11-12T09:31:35,748 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1645): Joining cluster... 2023-11-12T09:31:35,827 DEBUG [hconnection-0x39505ecb-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:31:35,830 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38566, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:31:35,849 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1657): Number of RegionServers=1 2023-11-12T09:31:35,849 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1699781555849 2023-11-12T09:31:35,850 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1699781615850 2023-11-12T09:31:35,850 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1664): Joined the cluster in 101 msec 2023-11-12T09:31:35,874 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35261,1699781489344-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:35,874 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35261,1699781489344-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:35,874 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35261,1699781489344-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:35,876 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:35261, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:35,877 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-11-12T09:31:35,883 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-11-12T09:31:35,899 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-11-12T09:31:35,900 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2369): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-12T09:31:35,910 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-11-12T09:31:35,913 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:31:35,915 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:35,917 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:31:36,335 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2e735f09d7b85fcf9fb65b6e7191fab4, NAME => 'hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f 2023-11-12T09:31:36,347 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:36,347 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 2e735f09d7b85fcf9fb65b6e7191fab4, disabling compactions & flushes 2023-11-12T09:31:36,347 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. 2023-11-12T09:31:36,347 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. 2023-11-12T09:31:36,347 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. after waiting 0 ms 2023-11-12T09:31:36,347 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. 2023-11-12T09:31:36,347 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. 2023-11-12T09:31:36,347 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2e735f09d7b85fcf9fb65b6e7191fab4: 2023-11-12T09:31:36,350 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:31:36,370 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1699781496352"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781496352"}]},"ts":"1699781496352"} 2023-11-12T09:31:36,404 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:31:36,406 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:31:36,412 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781496406"}]},"ts":"1699781496406"} 2023-11-12T09:31:36,415 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-11-12T09:31:36,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=2e735f09d7b85fcf9fb65b6e7191fab4, ASSIGN}] 2023-11-12T09:31:36,425 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=2e735f09d7b85fcf9fb65b6e7191fab4, ASSIGN 2023-11-12T09:31:36,426 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=2e735f09d7b85fcf9fb65b6e7191fab4, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37711,1699781492776; forceNewPlan=false, retain=false 2023-11-12T09:31:36,577 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=2e735f09d7b85fcf9fb65b6e7191fab4, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:36,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 2e735f09d7b85fcf9fb65b6e7191fab4, server=jenkins-hbase3.apache.org,37711,1699781492776}] 2023-11-12T09:31:36,734 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:36,740 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. 2023-11-12T09:31:36,741 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 2e735f09d7b85fcf9fb65b6e7191fab4, NAME => 'hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:31:36,742 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:31:36,742 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 2e735f09d7b85fcf9fb65b6e7191fab4 2023-11-12T09:31:36,742 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:36,742 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 2e735f09d7b85fcf9fb65b6e7191fab4 2023-11-12T09:31:36,742 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 2e735f09d7b85fcf9fb65b6e7191fab4 2023-11-12T09:31:36,744 INFO [StoreOpener-2e735f09d7b85fcf9fb65b6e7191fab4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 2e735f09d7b85fcf9fb65b6e7191fab4 2023-11-12T09:31:36,746 INFO [StoreOpener-2e735f09d7b85fcf9fb65b6e7191fab4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e735f09d7b85fcf9fb65b6e7191fab4 columnFamilyName info 2023-11-12T09:31:36,746 DEBUG [StoreOpener-2e735f09d7b85fcf9fb65b6e7191fab4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:36,747 INFO [StoreOpener-2e735f09d7b85fcf9fb65b6e7191fab4-1 {}] regionserver.HStore(324): Store=2e735f09d7b85fcf9fb65b6e7191fab4/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:31:36,749 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/namespace/2e735f09d7b85fcf9fb65b6e7191fab4 2023-11-12T09:31:36,749 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/namespace/2e735f09d7b85fcf9fb65b6e7191fab4 2023-11-12T09:31:36,752 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 2e735f09d7b85fcf9fb65b6e7191fab4 2023-11-12T09:31:36,756 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/namespace/2e735f09d7b85fcf9fb65b6e7191fab4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:31:36,757 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 2e735f09d7b85fcf9fb65b6e7191fab4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9460428000, jitterRate=-0.11892898380756378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:31:36,758 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 2e735f09d7b85fcf9fb65b6e7191fab4: 2023-11-12T09:31:36,760 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4., pid=6, masterSystemTime=1699781496734 2023-11-12T09:31:36,764 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. 2023-11-12T09:31:36,764 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. 2023-11-12T09:31:36,765 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=2e735f09d7b85fcf9fb65b6e7191fab4, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:36,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-11-12T09:31:36,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 2e735f09d7b85fcf9fb65b6e7191fab4, server=jenkins-hbase3.apache.org,37711,1699781492776 in 187 msec 2023-11-12T09:31:36,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-11-12T09:31:36,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=2e735f09d7b85fcf9fb65b6e7191fab4, ASSIGN in 350 msec 2023-11-12T09:31:36,776 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:31:36,777 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781496776"}]},"ts":"1699781496776"} 2023-11-12T09:31:36,784 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-11-12T09:31:36,788 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:31:36,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 887 msec 2023-11-12T09:31:36,814 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-11-12T09:31:36,816 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-11-12T09:31:36,816 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:36,817 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:31:36,864 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-11-12T09:31:36,880 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:31:36,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 30 msec 2023-11-12T09:31:36,891 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-11-12T09:31:36,901 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:31:36,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 20 msec 2023-11-12T09:31:36,927 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-11-12T09:31:36,931 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-11-12T09:31:36,934 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1194): Master has completed initialization 3.996sec 2023-11-12T09:31:36,936 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-11-12T09:31:36,937 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-11-12T09:31:36,938 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-11-12T09:31:36,939 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-11-12T09:31:36,939 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-11-12T09:31:36,941 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35261,1699781489344-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-12T09:31:36,942 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35261,1699781489344-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-11-12T09:31:36,955 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1297): Balancer post startup initialization complete, took 0 seconds 2023-11-12T09:31:36,956 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-11-12T09:31:37,014 DEBUG [Listener at localhost/32839 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x577a5d63 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:31:37,019 DEBUG [Listener at localhost/32839 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ea3473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:31:37,024 DEBUG [Listener at localhost/32839 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-11-12T09:31:37,024 DEBUG [Listener at localhost/32839 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-11-12T09:31:37,038 DEBUG [hconnection-0x556fb035-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:31:37,049 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38574, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:31:37,060 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,35261,1699781489344 2023-11-12T09:31:37,060 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-11-12T09:31:37,060 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/test.cache.data in system properties and HBase conf 2023-11-12T09:31:37,060 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop.tmp.dir in system properties and HBase conf 2023-11-12T09:31:37,060 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop.log.dir in system properties and HBase conf 2023-11-12T09:31:37,060 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-12T09:31:37,060 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-12T09:31:37,060 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-12T09:31:37,060 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:31:37,060 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:31:37,061 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-12T09:31:37,061 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:31:37,061 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-12T09:31:37,061 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-12T09:31:37,061 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:31:37,061 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:31:37,061 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-12T09:31:37,061 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/nfs.dump.dir in system properties and HBase conf 2023-11-12T09:31:37,061 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/java.io.tmpdir in system properties and HBase conf 2023-11-12T09:31:37,061 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:31:37,061 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-12T09:31:37,061 INFO [Listener at localhost/32839 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-11-12T09:31:38,951 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:31:39,333 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:31:39,340 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_36103_jobhistory____rtse2j/webapp 2023-11-12T09:31:39,507 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-11-12T09:31:39,507 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-11-12T09:31:39,507 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:31:39,511 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:31:39,599 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:31:39,749 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:31:40,121 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-11-12T09:31:40,172 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:36103 2023-11-12T09:31:40,354 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-11-12T09:31:40,356 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-11-12T09:31:40,447 WARN [Listener at jenkins-hbase3.apache.org/38427 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:31:40,528 WARN [Listener at jenkins-hbase3.apache.org/38427 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:31:40,529 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:31:40,538 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_36505_cluster____b96i30/webapp 2023-11-12T09:31:40,564 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-11-12T09:31:40,564 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-11-12T09:31:40,571 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-11-12T09:31:40,571 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-11-12T09:31:40,573 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:31:40,573 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-11-12T09:31:40,573 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-12T09:31:40,573 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-11-12T09:31:40,573 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-11-12T09:31:40,573 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-11-12T09:31:40,633 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:31:40,633 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-11-12T09:31:40,634 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:31:40,634 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:31:40,644 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:31:40,876 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:31:40,968 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:31:40,979 INFO [Listener at jenkins-hbase3.apache.org/38427 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:36505 2023-11-12T09:31:41,218 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:31:41,433 WARN [Listener at jenkins-hbase3.apache.org/44085 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:31:41,458 WARN [Listener at jenkins-hbase3.apache.org/44085 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:31:41,458 INFO [Listener at jenkins-hbase3.apache.org/44085 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:31:41,467 INFO [Listener at jenkins-hbase3.apache.org/44085 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_41313_node____8g68ev/webapp 2023-11-12T09:31:41,524 INFO [Listener at jenkins-hbase3.apache.org/44085 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-12T09:31:41,525 INFO [Listener at jenkins-hbase3.apache.org/44085 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:31:41,525 INFO [Listener at jenkins-hbase3.apache.org/44085 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:31:41,525 INFO [Listener at jenkins-hbase3.apache.org/44085 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:31:41,533 INFO [Listener at jenkins-hbase3.apache.org/44085 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:31:41,556 INFO [Listener at jenkins-hbase3.apache.org/44085 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:31:41,638 INFO [Listener at jenkins-hbase3.apache.org/44085 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:31:41,652 INFO [Listener at jenkins-hbase3.apache.org/44085 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:41313 2023-11-12T09:31:41,813 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:31:41,822 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:31:41,823 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:31:41,833 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_36559_node____.epxb4b/webapp 2023-11-12T09:31:41,886 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-12T09:31:41,886 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:31:41,886 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:31:41,887 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:31:41,894 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:31:41,910 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:31:41,975 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:31:41,981 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:36559 2023-11-12T09:31:42,011 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-11-12T09:31:42,011 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:38513/backupUT 2023-11-12T09:31:42,022 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:31:42,026 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57928, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:31:42,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-11-12T09:31:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-11-12T09:31:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-12T09:31:42,062 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:31:42,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 30 msec 2023-11-12T09:31:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-12T09:31:42,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-11-12T09:31:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-11-12T09:31:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-12T09:31:42,170 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:31:42,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 19 msec 2023-11-12T09:31:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-12T09:31:42,267 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-11-12T09:31:42,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-11-12T09:31:42,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-12T09:31:42,282 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:31:42,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 18 msec 2023-11-12T09:31:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-12T09:31:42,378 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-11-12T09:31:42,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-11-12T09:31:42,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-12T09:31:42,392 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:31:42,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 17 msec 2023-11-12T09:31:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-12T09:31:42,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'test-1699781502011', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:31:42,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1699781502011 2023-11-12T09:31:42,504 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1699781502011 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:31:42,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1699781502011" procId is: 13 2023-11-12T09:31:42,504 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:42,506 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1699781502011 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:31:42,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:31:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:31:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:31:42,939 INFO [RegionOpenAndInit-test-1699781502011-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d2b50154a764d8ff14c1aceb1c414835, NAME => 'test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1699781502011', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f 2023-11-12T09:31:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:31:43,358 DEBUG [RegionOpenAndInit-test-1699781502011-pool-0 {}] regionserver.HRegion(888): Instantiated test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:43,358 DEBUG [RegionOpenAndInit-test-1699781502011-pool-0 {}] regionserver.HRegion(1668): Closing d2b50154a764d8ff14c1aceb1c414835, disabling compactions & flushes 2023-11-12T09:31:43,358 INFO [RegionOpenAndInit-test-1699781502011-pool-0 {}] regionserver.HRegion(1690): Closing region test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:31:43,358 DEBUG [RegionOpenAndInit-test-1699781502011-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:31:43,358 DEBUG [RegionOpenAndInit-test-1699781502011-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. after waiting 0 ms 2023-11-12T09:31:43,358 DEBUG [RegionOpenAndInit-test-1699781502011-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:31:43,358 INFO [RegionOpenAndInit-test-1699781502011-pool-0 {}] regionserver.HRegion(1909): Closed test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:31:43,358 DEBUG [RegionOpenAndInit-test-1699781502011-pool-0 {}] regionserver.HRegion(1622): Region close journal for d2b50154a764d8ff14c1aceb1c414835: 2023-11-12T09:31:43,360 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1699781502011 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:31:43,360 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1699781503360"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781503360"}]},"ts":"1699781503360"} 2023-11-12T09:31:43,364 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:31:43,365 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1699781502011 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:31:43,365 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781502011","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781503365"}]},"ts":"1699781503365"} 2023-11-12T09:31:43,368 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781502011, state=ENABLING in hbase:meta 2023-11-12T09:31:43,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699781502011, region=d2b50154a764d8ff14c1aceb1c414835, ASSIGN}] 2023-11-12T09:31:43,376 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699781502011, region=d2b50154a764d8ff14c1aceb1c414835, ASSIGN 2023-11-12T09:31:43,377 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1699781502011, region=d2b50154a764d8ff14c1aceb1c414835, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37711,1699781492776; forceNewPlan=false, retain=false 2023-11-12T09:31:43,529 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=d2b50154a764d8ff14c1aceb1c414835, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:43,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure d2b50154a764d8ff14c1aceb1c414835, server=jenkins-hbase3.apache.org,37711,1699781492776}] 2023-11-12T09:31:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:31:43,686 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:43,694 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:31:43,694 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => d2b50154a764d8ff14c1aceb1c414835, NAME => 'test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:31:43,695 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:31:43,695 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1699781502011 d2b50154a764d8ff14c1aceb1c414835 2023-11-12T09:31:43,695 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:43,695 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for d2b50154a764d8ff14c1aceb1c414835 2023-11-12T09:31:43,695 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for d2b50154a764d8ff14c1aceb1c414835 2023-11-12T09:31:43,698 INFO [StoreOpener-d2b50154a764d8ff14c1aceb1c414835-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region d2b50154a764d8ff14c1aceb1c414835 2023-11-12T09:31:43,701 INFO [StoreOpener-d2b50154a764d8ff14c1aceb1c414835-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2b50154a764d8ff14c1aceb1c414835 columnFamilyName f 2023-11-12T09:31:43,701 DEBUG [StoreOpener-d2b50154a764d8ff14c1aceb1c414835-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:43,703 INFO [StoreOpener-d2b50154a764d8ff14c1aceb1c414835-1 {}] regionserver.HStore(324): Store=d2b50154a764d8ff14c1aceb1c414835/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:31:43,704 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835 2023-11-12T09:31:43,705 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835 2023-11-12T09:31:43,711 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for d2b50154a764d8ff14c1aceb1c414835 2023-11-12T09:31:43,720 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:31:43,721 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened d2b50154a764d8ff14c1aceb1c414835; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11059567200, jitterRate=0.03000245988368988}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:31:43,722 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for d2b50154a764d8ff14c1aceb1c414835: 2023-11-12T09:31:43,723 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835., pid=15, masterSystemTime=1699781503686 2023-11-12T09:31:43,726 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:31:43,726 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:31:43,727 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=d2b50154a764d8ff14c1aceb1c414835, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:43,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-11-12T09:31:43,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure d2b50154a764d8ff14c1aceb1c414835, server=jenkins-hbase3.apache.org,37711,1699781492776 in 198 msec 2023-11-12T09:31:43,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-11-12T09:31:43,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1699781502011, region=d2b50154a764d8ff14c1aceb1c414835, ASSIGN in 360 msec 2023-11-12T09:31:43,737 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1699781502011 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:31:43,738 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699781502011","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781503737"}]},"ts":"1699781503737"} 2023-11-12T09:31:43,740 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699781502011, state=ENABLED in hbase:meta 2023-11-12T09:31:43,743 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1699781502011 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:31:43,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1699781502011 in 1.2440 sec 2023-11-12T09:31:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:31:44,622 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: default:test-1699781502011, procId: 13 completed 2023-11-12T09:31:44,624 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x68b53090 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:31:44,632 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a564e0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:31:44,634 DEBUG [hconnection-0x23c3b760-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:31:44,639 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38580, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:31:44,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37711 {}] regionserver.HRegion(8210): writing data to region test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. with WAL disabled. Data may be lost in the event of a crash. 2023-11-12T09:31:44,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns2:test-16997815020111', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:31:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16997815020111 2023-11-12T09:31:44,774 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16997815020111 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:31:44,774 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:44,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16997815020111" procId is: 16 2023-11-12T09:31:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:31:44,776 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16997815020111 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:31:44,794 INFO [RegionOpenAndInit-ns2:test-16997815020111-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 59d601d3d6dbb1192a7f8ee12e4c9815, NAME => 'ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16997815020111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f 2023-11-12T09:31:44,811 DEBUG [RegionOpenAndInit-ns2:test-16997815020111-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:44,811 DEBUG [RegionOpenAndInit-ns2:test-16997815020111-pool-0 {}] regionserver.HRegion(1668): Closing 59d601d3d6dbb1192a7f8ee12e4c9815, disabling compactions & flushes 2023-11-12T09:31:44,811 INFO [RegionOpenAndInit-ns2:test-16997815020111-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:31:44,811 DEBUG [RegionOpenAndInit-ns2:test-16997815020111-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:31:44,811 DEBUG [RegionOpenAndInit-ns2:test-16997815020111-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. after waiting 0 ms 2023-11-12T09:31:44,811 DEBUG [RegionOpenAndInit-ns2:test-16997815020111-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:31:44,811 INFO [RegionOpenAndInit-ns2:test-16997815020111-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:31:44,812 DEBUG [RegionOpenAndInit-ns2:test-16997815020111-pool-0 {}] regionserver.HRegion(1622): Region close journal for 59d601d3d6dbb1192a7f8ee12e4c9815: 2023-11-12T09:31:44,813 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16997815020111 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:31:44,813 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781504813"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781504813"}]},"ts":"1699781504813"} 2023-11-12T09:31:44,816 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:31:44,817 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16997815020111 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:31:44,817 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997815020111","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781504817"}]},"ts":"1699781504817"} 2023-11-12T09:31:44,818 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997815020111, state=ENABLING in hbase:meta 2023-11-12T09:31:44,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997815020111, region=59d601d3d6dbb1192a7f8ee12e4c9815, ASSIGN}] 2023-11-12T09:31:44,831 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997815020111, region=59d601d3d6dbb1192a7f8ee12e4c9815, ASSIGN 2023-11-12T09:31:44,832 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16997815020111, region=59d601d3d6dbb1192a7f8ee12e4c9815, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37711,1699781492776; forceNewPlan=false, retain=false 2023-11-12T09:31:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:31:44,984 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=59d601d3d6dbb1192a7f8ee12e4c9815, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:44,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 59d601d3d6dbb1192a7f8ee12e4c9815, server=jenkins-hbase3.apache.org,37711,1699781492776}] 2023-11-12T09:31:45,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:31:45,140 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:45,146 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:31:45,146 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 59d601d3d6dbb1192a7f8ee12e4c9815, NAME => 'ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:31:45,146 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:31:45,147 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997815020111 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:31:45,147 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:45,147 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:31:45,147 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:31:45,150 INFO [StoreOpener-59d601d3d6dbb1192a7f8ee12e4c9815-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:31:45,152 INFO [StoreOpener-59d601d3d6dbb1192a7f8ee12e4c9815-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59d601d3d6dbb1192a7f8ee12e4c9815 columnFamilyName f 2023-11-12T09:31:45,152 DEBUG [StoreOpener-59d601d3d6dbb1192a7f8ee12e4c9815-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:45,153 INFO [StoreOpener-59d601d3d6dbb1192a7f8ee12e4c9815-1 {}] regionserver.HStore(324): Store=59d601d3d6dbb1192a7f8ee12e4c9815/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:31:45,154 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:31:45,155 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:31:45,158 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:31:45,165 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:31:45,166 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 59d601d3d6dbb1192a7f8ee12e4c9815; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11500636640, jitterRate=0.07108025252819061}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:31:45,167 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 59d601d3d6dbb1192a7f8ee12e4c9815: 2023-11-12T09:31:45,168 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815., pid=18, masterSystemTime=1699781505140 2023-11-12T09:31:45,171 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:31:45,171 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:31:45,172 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=59d601d3d6dbb1192a7f8ee12e4c9815, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:45,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-11-12T09:31:45,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 59d601d3d6dbb1192a7f8ee12e4c9815, server=jenkins-hbase3.apache.org,37711,1699781492776 in 187 msec 2023-11-12T09:31:45,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-11-12T09:31:45,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16997815020111, region=59d601d3d6dbb1192a7f8ee12e4c9815, ASSIGN in 349 msec 2023-11-12T09:31:45,182 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16997815020111 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:31:45,182 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997815020111","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781505182"}]},"ts":"1699781505182"} 2023-11-12T09:31:45,185 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997815020111, state=ENABLED in hbase:meta 2023-11-12T09:31:45,189 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16997815020111 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:31:45,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16997815020111 in 420 msec 2023-11-12T09:31:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:31:45,382 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns2:test-16997815020111, procId: 16 completed 2023-11-12T09:31:45,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37711 {}] regionserver.HRegion(8210): writing data to region ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. with WAL disabled. Data may be lost in the event of a crash. 2023-11-12T09:31:45,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns3:test-16997815020112', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:31:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16997815020112 2023-11-12T09:31:45,487 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16997815020112 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:31:45,487 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:45,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16997815020112" procId is: 19 2023-11-12T09:31:45,489 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16997815020112 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:31:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:31:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:31:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:31:45,914 INFO [RegionOpenAndInit-ns3:test-16997815020112-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 13632229548fde4afa1b285f20ec0adf, NAME => 'ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16997815020112', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f 2023-11-12T09:31:45,932 DEBUG [RegionOpenAndInit-ns3:test-16997815020112-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:45,932 DEBUG [RegionOpenAndInit-ns3:test-16997815020112-pool-0 {}] regionserver.HRegion(1668): Closing 13632229548fde4afa1b285f20ec0adf, disabling compactions & flushes 2023-11-12T09:31:45,932 INFO [RegionOpenAndInit-ns3:test-16997815020112-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:31:45,932 DEBUG [RegionOpenAndInit-ns3:test-16997815020112-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:31:45,932 DEBUG [RegionOpenAndInit-ns3:test-16997815020112-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. after waiting 0 ms 2023-11-12T09:31:45,932 DEBUG [RegionOpenAndInit-ns3:test-16997815020112-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:31:45,932 INFO [RegionOpenAndInit-ns3:test-16997815020112-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:31:45,932 DEBUG [RegionOpenAndInit-ns3:test-16997815020112-pool-0 {}] regionserver.HRegion(1622): Region close journal for 13632229548fde4afa1b285f20ec0adf: 2023-11-12T09:31:45,934 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16997815020112 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:31:45,934 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781505934"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781505934"}]},"ts":"1699781505934"} 2023-11-12T09:31:45,937 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:31:45,938 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16997815020112 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:31:45,938 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997815020112","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781505938"}]},"ts":"1699781505938"} 2023-11-12T09:31:45,940 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997815020112, state=ENABLING in hbase:meta 2023-11-12T09:31:45,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997815020112, region=13632229548fde4afa1b285f20ec0adf, ASSIGN}] 2023-11-12T09:31:45,950 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997815020112, region=13632229548fde4afa1b285f20ec0adf, ASSIGN 2023-11-12T09:31:45,951 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16997815020112, region=13632229548fde4afa1b285f20ec0adf, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37711,1699781492776; forceNewPlan=false, retain=false 2023-11-12T09:31:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:31:46,103 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=13632229548fde4afa1b285f20ec0adf, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:46,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 13632229548fde4afa1b285f20ec0adf, server=jenkins-hbase3.apache.org,37711,1699781492776}] 2023-11-12T09:31:46,259 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:46,264 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:31:46,264 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 13632229548fde4afa1b285f20ec0adf, NAME => 'ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:31:46,265 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:31:46,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997815020112 13632229548fde4afa1b285f20ec0adf 2023-11-12T09:31:46,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:46,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 13632229548fde4afa1b285f20ec0adf 2023-11-12T09:31:46,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 13632229548fde4afa1b285f20ec0adf 2023-11-12T09:31:46,267 INFO [StoreOpener-13632229548fde4afa1b285f20ec0adf-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 13632229548fde4afa1b285f20ec0adf 2023-11-12T09:31:46,269 INFO [StoreOpener-13632229548fde4afa1b285f20ec0adf-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13632229548fde4afa1b285f20ec0adf columnFamilyName f 2023-11-12T09:31:46,270 DEBUG [StoreOpener-13632229548fde4afa1b285f20ec0adf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:46,270 INFO [StoreOpener-13632229548fde4afa1b285f20ec0adf-1 {}] regionserver.HStore(324): Store=13632229548fde4afa1b285f20ec0adf/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:31:46,272 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns3/test-16997815020112/13632229548fde4afa1b285f20ec0adf 2023-11-12T09:31:46,272 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns3/test-16997815020112/13632229548fde4afa1b285f20ec0adf 2023-11-12T09:31:46,276 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 13632229548fde4afa1b285f20ec0adf 2023-11-12T09:31:46,279 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns3/test-16997815020112/13632229548fde4afa1b285f20ec0adf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:31:46,280 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 13632229548fde4afa1b285f20ec0adf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10960862400, jitterRate=0.020809859037399292}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:31:46,280 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 13632229548fde4afa1b285f20ec0adf: 2023-11-12T09:31:46,281 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf., pid=21, masterSystemTime=1699781506258 2023-11-12T09:31:46,284 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:31:46,284 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:31:46,285 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=13632229548fde4afa1b285f20ec0adf, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:46,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-11-12T09:31:46,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 13632229548fde4afa1b285f20ec0adf, server=jenkins-hbase3.apache.org,37711,1699781492776 in 182 msec 2023-11-12T09:31:46,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-11-12T09:31:46,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16997815020112, region=13632229548fde4afa1b285f20ec0adf, ASSIGN in 342 msec 2023-11-12T09:31:46,294 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16997815020112 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:31:46,295 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997815020112","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781506294"}]},"ts":"1699781506294"} 2023-11-12T09:31:46,297 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997815020112, state=ENABLED in hbase:meta 2023-11-12T09:31:46,300 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16997815020112 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:31:46,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16997815020112 in 817 msec 2023-11-12T09:31:46,357 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16997815020111' 2023-11-12T09:31:46,358 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16997815020112' 2023-11-12T09:31:46,358 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1699781502011' 2023-11-12T09:31:46,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:31:46,599 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns3:test-16997815020112, procId: 19 completed 2023-11-12T09:31:46,599 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16997815020112 get assigned. Timeout = 60000ms 2023-11-12T09:31:46,602 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:31:46,607 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16997815020112 assigned to meta. Checking AM states. 2023-11-12T09:31:46,608 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:31:46,608 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16997815020112 assigned. 2023-11-12T09:31:46,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns4:test-16997815020113', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:31:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16997815020113 2023-11-12T09:31:46,615 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16997815020113 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:31:46,615 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:46,615 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16997815020113" procId is: 22 2023-11-12T09:31:46,616 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16997815020113 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:31:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:31:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:31:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:31:47,032 INFO [RegionOpenAndInit-ns4:test-16997815020113-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => f37756824c9283a5488eef653b620975, NAME => 'ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16997815020113', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f 2023-11-12T09:31:47,049 DEBUG [RegionOpenAndInit-ns4:test-16997815020113-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:47,049 DEBUG [RegionOpenAndInit-ns4:test-16997815020113-pool-0 {}] regionserver.HRegion(1668): Closing f37756824c9283a5488eef653b620975, disabling compactions & flushes 2023-11-12T09:31:47,050 INFO [RegionOpenAndInit-ns4:test-16997815020113-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. 2023-11-12T09:31:47,050 DEBUG [RegionOpenAndInit-ns4:test-16997815020113-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. 2023-11-12T09:31:47,050 DEBUG [RegionOpenAndInit-ns4:test-16997815020113-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. after waiting 0 ms 2023-11-12T09:31:47,050 DEBUG [RegionOpenAndInit-ns4:test-16997815020113-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. 2023-11-12T09:31:47,050 INFO [RegionOpenAndInit-ns4:test-16997815020113-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. 2023-11-12T09:31:47,050 DEBUG [RegionOpenAndInit-ns4:test-16997815020113-pool-0 {}] regionserver.HRegion(1622): Region close journal for f37756824c9283a5488eef653b620975: 2023-11-12T09:31:47,052 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16997815020113 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:31:47,052 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699781507052"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781507052"}]},"ts":"1699781507052"} 2023-11-12T09:31:47,055 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:31:47,056 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16997815020113 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:31:47,056 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16997815020113","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781507056"}]},"ts":"1699781507056"} 2023-11-12T09:31:47,058 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16997815020113, state=ENABLING in hbase:meta 2023-11-12T09:31:47,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16997815020113, region=f37756824c9283a5488eef653b620975, ASSIGN}] 2023-11-12T09:31:47,066 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16997815020113, region=f37756824c9283a5488eef653b620975, ASSIGN 2023-11-12T09:31:47,068 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16997815020113, region=f37756824c9283a5488eef653b620975, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37711,1699781492776; forceNewPlan=false, retain=false 2023-11-12T09:31:47,220 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=f37756824c9283a5488eef653b620975, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:31:47,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure f37756824c9283a5488eef653b620975, server=jenkins-hbase3.apache.org,37711,1699781492776}] 2023-11-12T09:31:47,375 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:47,380 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. 2023-11-12T09:31:47,381 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => f37756824c9283a5488eef653b620975, NAME => 'ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:31:47,381 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:31:47,381 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997815020113 f37756824c9283a5488eef653b620975 2023-11-12T09:31:47,381 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:47,382 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for f37756824c9283a5488eef653b620975 2023-11-12T09:31:47,382 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for f37756824c9283a5488eef653b620975 2023-11-12T09:31:47,384 INFO [StoreOpener-f37756824c9283a5488eef653b620975-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f37756824c9283a5488eef653b620975 2023-11-12T09:31:47,386 INFO [StoreOpener-f37756824c9283a5488eef653b620975-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f37756824c9283a5488eef653b620975 columnFamilyName f 2023-11-12T09:31:47,386 DEBUG [StoreOpener-f37756824c9283a5488eef653b620975-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:47,387 INFO [StoreOpener-f37756824c9283a5488eef653b620975-1 {}] regionserver.HStore(324): Store=f37756824c9283a5488eef653b620975/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:31:47,388 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns4/test-16997815020113/f37756824c9283a5488eef653b620975 2023-11-12T09:31:47,389 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns4/test-16997815020113/f37756824c9283a5488eef653b620975 2023-11-12T09:31:47,393 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for f37756824c9283a5488eef653b620975 2023-11-12T09:31:47,396 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns4/test-16997815020113/f37756824c9283a5488eef653b620975/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:31:47,397 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened f37756824c9283a5488eef653b620975; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10328580000, jitterRate=-0.03807602822780609}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:31:47,397 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for f37756824c9283a5488eef653b620975: 2023-11-12T09:31:47,399 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975., pid=24, masterSystemTime=1699781507375 2023-11-12T09:31:47,401 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. 2023-11-12T09:31:47,401 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. 2023-11-12T09:31:47,402 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=f37756824c9283a5488eef653b620975, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:47,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-11-12T09:31:47,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure f37756824c9283a5488eef653b620975, server=jenkins-hbase3.apache.org,37711,1699781492776 in 183 msec 2023-11-12T09:31:47,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-11-12T09:31:47,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16997815020113, region=f37756824c9283a5488eef653b620975, ASSIGN in 343 msec 2023-11-12T09:31:47,411 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16997815020113 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:31:47,412 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16997815020113","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781507411"}]},"ts":"1699781507411"} 2023-11-12T09:31:47,414 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16997815020113, state=ENABLED in hbase:meta 2023-11-12T09:31:47,418 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16997815020113 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:31:47,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16997815020113 in 807 msec 2023-11-12T09:31:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:31:47,725 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns4:test-16997815020113, procId: 22 completed 2023-11-12T09:31:47,725 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16997815020113 get assigned. Timeout = 60000ms 2023-11-12T09:31:47,725 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:31:47,729 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16997815020113 assigned to meta. Checking AM states. 2023-11-12T09:31:47,729 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:31:47,729 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16997815020113 assigned. 2023-11-12T09:31:47,729 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x68b53090 to 127.0.0.1:55425 2023-11-12T09:31:47,730 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:47,774 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.ResourceChecker(147): before: backup.TestBackupMultipleDeletes#testBackupMultipleDeletes Thread=584, OpenFileDescriptor=689, MaxFileDescriptor=60000, SystemLoadAverage=254, ProcessCount=168, AvailableMemoryMB=6194 2023-11-12T09:31:47,774 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.ResourceChecker(130): Thread=584 is superior to 500 2023-11-12T09:31:47,776 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] backup.TestBackupMultipleDeletes(60): create full backup image for all tables 2023-11-12T09:31:47,778 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4539b7f4 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:31:47,784 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bfe60e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:31:47,794 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:31:47,795 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:40784, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:31:47,808 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:31:47,809 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:60698, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:31:47,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-11-12T09:31:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-11-12T09:31:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-12T09:31:47,829 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:31:47,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 15 msec 2023-11-12T09:31:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-12T09:31:47,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:31:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-11-12T09:31:47,945 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:31:47,945 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:47,945 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-11-12T09:31:47,946 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:31:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:31:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:31:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:31:48,365 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 0e93acef4885d53d7a3e4dd8fbdd16bb, NAME => 'backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f 2023-11-12T09:31:48,376 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:48,377 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 0e93acef4885d53d7a3e4dd8fbdd16bb, disabling compactions & flushes 2023-11-12T09:31:48,377 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:31:48,377 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:31:48,377 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. after waiting 0 ms 2023-11-12T09:31:48,377 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:31:48,377 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:31:48,377 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 0e93acef4885d53d7a3e4dd8fbdd16bb: 2023-11-12T09:31:48,379 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:31:48,380 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1699781508379"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781508379"}]},"ts":"1699781508379"} 2023-11-12T09:31:48,382 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:31:48,383 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:31:48,383 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781508383"}]},"ts":"1699781508383"} 2023-11-12T09:31:48,385 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-12T09:31:48,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=0e93acef4885d53d7a3e4dd8fbdd16bb, ASSIGN}] 2023-11-12T09:31:48,392 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=0e93acef4885d53d7a3e4dd8fbdd16bb, ASSIGN 2023-11-12T09:31:48,394 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=0e93acef4885d53d7a3e4dd8fbdd16bb, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37711,1699781492776; forceNewPlan=false, retain=false 2023-11-12T09:31:48,545 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=0e93acef4885d53d7a3e4dd8fbdd16bb, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:31:48,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb, server=jenkins-hbase3.apache.org,37711,1699781492776}] 2023-11-12T09:31:48,703 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:48,708 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:31:48,708 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 0e93acef4885d53d7a3e4dd8fbdd16bb, NAME => 'backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:31:48,709 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:31:48,709 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:31:48,709 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:48,709 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:31:48,709 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:31:48,711 INFO [StoreOpener-0e93acef4885d53d7a3e4dd8fbdd16bb-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:31:48,714 INFO [StoreOpener-0e93acef4885d53d7a3e4dd8fbdd16bb-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e93acef4885d53d7a3e4dd8fbdd16bb columnFamilyName meta 2023-11-12T09:31:48,714 DEBUG [StoreOpener-0e93acef4885d53d7a3e4dd8fbdd16bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:48,714 INFO [StoreOpener-0e93acef4885d53d7a3e4dd8fbdd16bb-1 {}] regionserver.HStore(324): Store=0e93acef4885d53d7a3e4dd8fbdd16bb/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:31:48,715 INFO [StoreOpener-0e93acef4885d53d7a3e4dd8fbdd16bb-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:31:48,716 INFO [StoreOpener-0e93acef4885d53d7a3e4dd8fbdd16bb-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e93acef4885d53d7a3e4dd8fbdd16bb columnFamilyName session 2023-11-12T09:31:48,717 DEBUG [StoreOpener-0e93acef4885d53d7a3e4dd8fbdd16bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:48,717 INFO [StoreOpener-0e93acef4885d53d7a3e4dd8fbdd16bb-1 {}] regionserver.HStore(324): Store=0e93acef4885d53d7a3e4dd8fbdd16bb/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:31:48,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:31:48,719 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:31:48,720 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:31:48,722 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:31:48,725 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:31:48,726 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 0e93acef4885d53d7a3e4dd8fbdd16bb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10382687840, jitterRate=-0.033036842942237854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:31:48,726 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 0e93acef4885d53d7a3e4dd8fbdd16bb: 2023-11-12T09:31:48,727 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb., pid=28, masterSystemTime=1699781508703 2023-11-12T09:31:48,729 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:31:48,730 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:31:48,730 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=0e93acef4885d53d7a3e4dd8fbdd16bb, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:48,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-11-12T09:31:48,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb, server=jenkins-hbase3.apache.org,37711,1699781492776 in 181 msec 2023-11-12T09:31:48,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-11-12T09:31:48,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=0e93acef4885d53d7a3e4dd8fbdd16bb, ASSIGN in 345 msec 2023-11-12T09:31:48,738 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:31:48,739 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781508738"}]},"ts":"1699781508738"} 2023-11-12T09:31:48,740 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-12T09:31:48,743 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:31:48,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 803 msec 2023-11-12T09:31:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:31:49,054 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-11-12T09:31:49,068 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:31:49,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-11-12T09:31:49,072 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:31:49,072 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:49,072 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-11-12T09:31:49,073 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:31:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:31:49,085 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 5d178aff93cac37e2dcbca54eac41805, NAME => 'backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f 2023-11-12T09:31:49,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:31:49,204 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:31:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:31:49,496 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:49,496 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 5d178aff93cac37e2dcbca54eac41805, disabling compactions & flushes 2023-11-12T09:31:49,496 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. 2023-11-12T09:31:49,496 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. 2023-11-12T09:31:49,496 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. after waiting 0 ms 2023-11-12T09:31:49,496 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. 2023-11-12T09:31:49,496 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. 2023-11-12T09:31:49,496 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 5d178aff93cac37e2dcbca54eac41805: 2023-11-12T09:31:49,498 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:31:49,498 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1699781509498"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699781509498"}]},"ts":"1699781509498"} 2023-11-12T09:31:49,500 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:31:49,501 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:31:49,502 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781509501"}]},"ts":"1699781509501"} 2023-11-12T09:31:49,503 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-11-12T09:31:49,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=5d178aff93cac37e2dcbca54eac41805, ASSIGN}] 2023-11-12T09:31:49,512 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=5d178aff93cac37e2dcbca54eac41805, ASSIGN 2023-11-12T09:31:49,513 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=5d178aff93cac37e2dcbca54eac41805, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37711,1699781492776; forceNewPlan=false, retain=false 2023-11-12T09:31:49,665 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=5d178aff93cac37e2dcbca54eac41805, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:49,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 5d178aff93cac37e2dcbca54eac41805, server=jenkins-hbase3.apache.org,37711,1699781492776}] 2023-11-12T09:31:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:31:49,820 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:49,824 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. 2023-11-12T09:31:49,824 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 5d178aff93cac37e2dcbca54eac41805, NAME => 'backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:31:49,825 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:31:49,825 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 5d178aff93cac37e2dcbca54eac41805 2023-11-12T09:31:49,825 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:31:49,825 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 5d178aff93cac37e2dcbca54eac41805 2023-11-12T09:31:49,825 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 5d178aff93cac37e2dcbca54eac41805 2023-11-12T09:31:49,827 INFO [StoreOpener-5d178aff93cac37e2dcbca54eac41805-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 5d178aff93cac37e2dcbca54eac41805 2023-11-12T09:31:49,828 INFO [StoreOpener-5d178aff93cac37e2dcbca54eac41805-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d178aff93cac37e2dcbca54eac41805 columnFamilyName meta 2023-11-12T09:31:49,828 DEBUG [StoreOpener-5d178aff93cac37e2dcbca54eac41805-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:49,829 INFO [StoreOpener-5d178aff93cac37e2dcbca54eac41805-1 {}] regionserver.HStore(324): Store=5d178aff93cac37e2dcbca54eac41805/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:31:49,829 INFO [StoreOpener-5d178aff93cac37e2dcbca54eac41805-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 5d178aff93cac37e2dcbca54eac41805 2023-11-12T09:31:49,831 INFO [StoreOpener-5d178aff93cac37e2dcbca54eac41805-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d178aff93cac37e2dcbca54eac41805 columnFamilyName session 2023-11-12T09:31:49,831 DEBUG [StoreOpener-5d178aff93cac37e2dcbca54eac41805-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:31:49,831 INFO [StoreOpener-5d178aff93cac37e2dcbca54eac41805-1 {}] regionserver.HStore(324): Store=5d178aff93cac37e2dcbca54eac41805/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:31:49,832 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system_bulk/5d178aff93cac37e2dcbca54eac41805 2023-11-12T09:31:49,833 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system_bulk/5d178aff93cac37e2dcbca54eac41805 2023-11-12T09:31:49,834 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:31:49,836 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 5d178aff93cac37e2dcbca54eac41805 2023-11-12T09:31:49,838 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system_bulk/5d178aff93cac37e2dcbca54eac41805/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:31:49,839 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 5d178aff93cac37e2dcbca54eac41805; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11638912320, jitterRate=0.08395817875862122}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:31:49,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 5d178aff93cac37e2dcbca54eac41805: 2023-11-12T09:31:49,841 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805., pid=31, masterSystemTime=1699781509819 2023-11-12T09:31:49,843 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. 2023-11-12T09:31:49,843 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. 2023-11-12T09:31:49,844 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=5d178aff93cac37e2dcbca54eac41805, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:49,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-11-12T09:31:49,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 5d178aff93cac37e2dcbca54eac41805, server=jenkins-hbase3.apache.org,37711,1699781492776 in 179 msec 2023-11-12T09:31:49,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-11-12T09:31:49,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=5d178aff93cac37e2dcbca54eac41805, ASSIGN in 338 msec 2023-11-12T09:31:49,851 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:31:49,852 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699781509852"}]},"ts":"1699781509852"} 2023-11-12T09:31:49,853 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-11-12T09:31:49,857 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:31:49,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 789 msec 2023-11-12T09:31:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:31:50,181 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-11-12T09:31:50,194 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:31:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37711 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2023-11-12T09:31:50,229 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:31:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781510229 (current time:1699781510229). 2023-11-12T09:31:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:31:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:31:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2313a73d to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:31:50,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e42220b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:31:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:31:50,244 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:40796, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:31:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2313a73d to 127.0.0.1:55425 2023-11-12T09:31:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:50,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:31:50,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:31:50,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-12T09:31:50,262 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:31:50,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:31:50,268 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:31:50,280 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot, current working dir path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:31:50,282 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:31:50,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:31:50,563 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-12T09:31:50,563 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-11-12T09:31:50,564 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997815020111 2023-11-12T09:31:50,564 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997815020111 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:31:50,565 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16997815020112 2023-11-12T09:31:50,565 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16997815020112 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:31:50,566 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-11-12T09:31:50,566 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-11-12T09:31:50,566 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1699781502011 2023-11-12T09:31:50,566 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1699781502011 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:31:50,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:31:50,567 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16997815020113 2023-11-12T09:31:50,567 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16997815020113 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:31:50,695 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:31:50,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb}] 2023-11-12T09:31:50,706 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:31:50,868 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:31:50,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37711 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-11-12T09:31:50,870 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:31:50,872 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 0e93acef4885d53d7a3e4dd8fbdd16bb 2/2 column families, dataSize=45 B heapSize=632 B 2023-11-12T09:31:50,961 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/d07cc77bb3f3424098b0a6f3fd359a0f is 49, key is activesession:/session:c/1699781510215/Put/seqid=0 2023-11-12T09:31:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:31:51,386 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/d07cc77bb3f3424098b0a6f3fd359a0f 2023-11-12T09:31:51,477 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/d07cc77bb3f3424098b0a6f3fd359a0f as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f 2023-11-12T09:31:51,487 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f, entries=1, sequenceid=5, filesize=4.9 K 2023-11-12T09:31:51,492 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 0e93acef4885d53d7a3e4dd8fbdd16bb in 620ms, sequenceid=5, compaction requested=false 2023-11-12T09:31:51,493 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-12T09:31:51,495 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 0e93acef4885d53d7a3e4dd8fbdd16bb: 2023-11-12T09:31:51,495 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. for snapshot_backup_system completed. 2023-11-12T09:31:51,496 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:31:51,506 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:31:51,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-12T09:31:51,513 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f] hfiles 2023-11-12T09:31:51,513 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f for snapshot=snapshot_backup_system 2023-11-12T09:31:51,532 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:31:51,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-11-12T09:31:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4026): Remote procedure done, pid=33 2023-11-12T09:31:51,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:31:51,549 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:31:51,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-11-12T09:31:51,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb in 853 msec 2023-11-12T09:31:51,553 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:31:51,555 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:31:51,559 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:31:51,559 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:31:51,561 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:31:51,988 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:31:52,003 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:31:52,004 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:31:52,006 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:31:52,007 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-12T09:31:52,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.7520 sec 2023-11-12T09:31:52,360 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-11-12T09:31:52,360 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16997815020113' 2023-11-12T09:31:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:31:52,373 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-11-12T09:31:52,373 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(115): Backup backup_1699781507788 started at 1699781512373. 2023-11-12T09:31:52,395 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(119): Backup session backup_1699781507788 has been started. 2023-11-12T09:31:52,398 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:31:52,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:31:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:31:52,413 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:31:52,413 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:31:52,414 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:31:52,414 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:31:52,414 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:31:52,414 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:31:52,417 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:31:52,417 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,417 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:31:52,417 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:31:52,418 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,418 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:31:52,418 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:31:52,418 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:31:52,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:31:52,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:31:52,419 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:31:52,420 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:31:52,423 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:31:52,423 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:31:52,423 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:31:52,424 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:31:52,424 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:31:52,424 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:31:52,426 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,426 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:31:52,426 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,426 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:31:52,426 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:31:52,427 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:31:52,427 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:31:52,427 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:31:52,427 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:31:52,428 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:31:52,428 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,429 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:31:52,429 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:31:52,429 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2f8bfeed[Count = 0] remaining members to acquire global barrier 2023-11-12T09:31:52,429 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:31:52,429 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:31:52,431 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:31:52,431 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:31:52,431 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:31:52,431 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:31:52,432 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,432 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:31:52,432 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:31:52,433 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,433 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781494554 highest: 1699781494554 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,433 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776:(num 1699781494554) roll requested 2023-11-12T09:31:52,445 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781512433, exclude list is [], retry=0 2023-11-12T09:31:52,451 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:31:52,455 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781494554 with entries=19, filesize=4.02 KB; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781512433 2023-11-12T09:31:52,456 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:31:52,456 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781494554 is not closed yet, will try archiving it next time 2023-11-12T09:31:52,457 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta:.meta(num 1699781495617) roll requested 2023-11-12T09:31:52,468 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781512457.meta, exclude list is [], retry=0 2023-11-12T09:31:52,472 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:31:52,474 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-11-12T09:31:52,474 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781512433 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,475 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781495617.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781512457.meta 2023-11-12T09:31:52,475 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:31:52,475 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:31:52,475 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781495617.meta is not closed yet, will try archiving it next time 2023-11-12T09:31:52,477 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:60702, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:31:52,497 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:31:52,497 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37711,1699781492776' in zk 2023-11-12T09:31:52,500 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:31:52,500 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,500 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:31:52,500 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:31:52,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:31:52,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:31:52,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:31:52,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:31:52,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:31:52,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,503 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:31:52,503 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:31:52,503 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,504 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37711,1699781492776': 2023-11-12T09:31:52,505 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:31:52,505 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:31:52,505 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:31:52,505 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:31:52,505 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:31:52,505 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:31:52,508 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:31:52,508 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:31:52,508 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:31:52,508 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:31:52,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:31:52,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:31:52,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:31:52,509 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:31:52,509 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:31:52,509 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:31:52,509 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:31:52,509 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,509 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:31:52,510 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:31:52,510 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:31:52,511 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:31:52,511 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,512 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,512 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:31:52,512 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:31:52,513 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,522 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,522 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:31:52,522 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:31:52,522 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:31:52,522 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:31:52,522 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:31:52,522 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:52,523 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:31:52,523 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:31:52,523 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:31:52,523 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:31:52,523 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:31:52,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:31:52,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:31:52,524 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:31:52,524 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:31:52,525 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-12T09:31:52,526 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:31:52,626 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:31:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:31:52,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 } 2023-11-12T09:31:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781512636 (current time:1699781512636). 2023-11-12T09:31:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:31:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781512635_default_test-1699781502011 VERSION not specified, setting to 2 2023-11-12T09:31:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4ea7ac13 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:31:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d1308c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:31:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:31:52,648 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:40804, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:31:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4ea7ac13 to 127.0.0.1:55425 2023-11-12T09:31:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:31:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 } 2023-11-12T09:31:52,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-11-12T09:31:52,653 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:31:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:31:52,654 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:31:52,657 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot, current working dir path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_1699781512635_default_test-1699781502011 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:31:52,658 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:31:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:31:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:31:53,069 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:31:53,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure d2b50154a764d8ff14c1aceb1c414835}] 2023-11-12T09:31:53,072 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure d2b50154a764d8ff14c1aceb1c414835 2023-11-12T09:31:53,223 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:53,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37711 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-11-12T09:31:53,224 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:31:53,225 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing d2b50154a764d8ff14c1aceb1c414835 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-12T09:31:53,245 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/.tmp/f/f8fc91fc02ac435eb7492eb4828be543 is 37, key is row10/f:q1/1699781504658/Put/seqid=0 2023-11-12T09:31:53,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:31:53,656 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/.tmp/f/f8fc91fc02ac435eb7492eb4828be543 2023-11-12T09:31:53,668 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/.tmp/f/f8fc91fc02ac435eb7492eb4828be543 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/f/f8fc91fc02ac435eb7492eb4828be543 2023-11-12T09:31:53,680 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/f/f8fc91fc02ac435eb7492eb4828be543, entries=99, sequenceid=103, filesize=8.2 K 2023-11-12T09:31:53,683 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for d2b50154a764d8ff14c1aceb1c414835 in 457ms, sequenceid=103, compaction requested=false 2023-11-12T09:31:53,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for d2b50154a764d8ff14c1aceb1c414835: 2023-11-12T09:31:53,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. for snapshot_1699781512635_default_test-1699781502011 completed. 2023-11-12T09:31:53,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835.' region-info for snapshot=snapshot_1699781512635_default_test-1699781502011 2023-11-12T09:31:53,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:31:53,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/f/f8fc91fc02ac435eb7492eb4828be543] hfiles 2023-11-12T09:31:53,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/f/f8fc91fc02ac435eb7492eb4828be543 for snapshot=snapshot_1699781512635_default_test-1699781502011 2023-11-12T09:31:53,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:31:53,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-11-12T09:31:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4026): Remote procedure done, pid=35 2023-11-12T09:31:53,697 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781512635_default_test-1699781502011 on region d2b50154a764d8ff14c1aceb1c414835 2023-11-12T09:31:53,698 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure d2b50154a764d8ff14c1aceb1c414835 2023-11-12T09:31:53,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-11-12T09:31:53,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure d2b50154a764d8ff14c1aceb1c414835 in 630 msec 2023-11-12T09:31:53,701 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:31:53,702 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:31:53,703 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:31:53,703 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781512635_default_test-1699781502011 2023-11-12T09:31:53,704 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_1699781512635_default_test-1699781502011 2023-11-12T09:31:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:31:54,126 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:31:54,133 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:31:54,134 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_1699781512635_default_test-1699781502011 to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_1699781512635_default_test-1699781502011 2023-11-12T09:31:54,136 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:31:54,136 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-11-12T09:31:54,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699781512635_default_test-1699781502011 table=test-1699781502011 type=FLUSH ttl=0 } in 1.4860 sec 2023-11-12T09:31:54,214 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:31:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:31:54,763 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: default:test-1699781502011, procId: 34 completed 2023-11-12T09:31:54,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 } 2023-11-12T09:31:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781514765 (current time:1699781514765). 2023-11-12T09:31:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:31:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781514763_ns2_test-16997815020111 VERSION not specified, setting to 2 2023-11-12T09:31:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6d82d2e5 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:31:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eb63c72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:31:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:31:54,775 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:40806, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:31:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6d82d2e5 to 127.0.0.1:55425 2023-11-12T09:31:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:31:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:31:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 } 2023-11-12T09:31:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-11-12T09:31:54,782 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:31:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:31:54,783 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:31:54,788 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot, current working dir path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_1699781514763_ns2_test-16997815020111 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:31:54,791 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:31:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:31:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:31:55,205 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:31:55,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 59d601d3d6dbb1192a7f8ee12e4c9815}] 2023-11-12T09:31:55,208 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:31:55,359 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:31:55,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37711 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-11-12T09:31:55,360 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:31:55,361 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2819): Flushing 59d601d3d6dbb1192a7f8ee12e4c9815 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-12T09:31:55,379 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/.tmp/f/64a8b4f728734a48b016e4bd27b30a70 is 37, key is row10/f:q1/1699781505398/Put/seqid=0 2023-11-12T09:31:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:31:55,788 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/.tmp/f/64a8b4f728734a48b016e4bd27b30a70 2023-11-12T09:31:55,798 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/.tmp/f/64a8b4f728734a48b016e4bd27b30a70 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/f/64a8b4f728734a48b016e4bd27b30a70 2023-11-12T09:31:55,806 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/f/64a8b4f728734a48b016e4bd27b30a70, entries=99, sequenceid=103, filesize=8.2 K 2023-11-12T09:31:55,808 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 59d601d3d6dbb1192a7f8ee12e4c9815 in 447ms, sequenceid=103, compaction requested=false 2023-11-12T09:31:55,809 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 59d601d3d6dbb1192a7f8ee12e4c9815: 2023-11-12T09:31:55,809 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. for snapshot_1699781514763_ns2_test-16997815020111 completed. 2023-11-12T09:31:55,809 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815.' region-info for snapshot=snapshot_1699781514763_ns2_test-16997815020111 2023-11-12T09:31:55,809 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:31:55,809 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/f/64a8b4f728734a48b016e4bd27b30a70] hfiles 2023-11-12T09:31:55,809 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/f/64a8b4f728734a48b016e4bd27b30a70 for snapshot=snapshot_1699781514763_ns2_test-16997815020111 2023-11-12T09:31:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:31:56,220 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:31:56,220 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-11-12T09:31:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4026): Remote procedure done, pid=37 2023-11-12T09:31:56,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781514763_ns2_test-16997815020111 on region 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:31:56,222 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:31:56,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-11-12T09:31:56,225 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:31:56,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 59d601d3d6dbb1192a7f8ee12e4c9815 in 1.0180 sec 2023-11-12T09:31:56,226 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:31:56,227 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:31:56,227 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781514763_ns2_test-16997815020111 2023-11-12T09:31:56,228 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_1699781514763_ns2_test-16997815020111 2023-11-12T09:31:56,649 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:31:56,657 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:31:56,657 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_1699781514763_ns2_test-16997815020111 to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_1699781514763_ns2_test-16997815020111 2023-11-12T09:31:56,659 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:31:56,659 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-11-12T09:31:56,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1699781514763_ns2_test-16997815020111 table=ns2:test-16997815020111 type=FLUSH ttl=0 } in 1.8810 sec 2023-11-12T09:31:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:31:56,891 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: ns2:test-16997815020111, procId: 36 completed 2023-11-12T09:31:56,891 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781507788 2023-11-12T09:31:56,891 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:31:56,893 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-12T09:31:56,896 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781507788_ns2:test-16997815020111 2023-11-12T09:31:56,896 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781514763_ns2_test-16997815020111 to hdfs://localhost:38513/backupUT/backup_1699781507788/ns2/test-16997815020111/ 2023-11-12T09:31:56,906 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:31:56,941 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:38513, inputRoot=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f 2023-11-12T09:31:56,941 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,35261,1699781489344_-171988317_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38513/backupUT/backup_1699781507788/ns2/test-16997815020111, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38513/backupUT/backup_1699781507788/ns2/test-16997815020111/.hbase-snapshot/.tmp/snapshot_1699781514763_ns2_test-16997815020111 2023-11-12T09:31:56,941 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:38513, inputRoot=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f, snapshotDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_1699781514763_ns2_test-16997815020111. 2023-11-12T09:31:56,956 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_1699781514763_ns2_test-16997815020111 to hdfs://localhost:38513/backupUT/backup_1699781507788/ns2/test-16997815020111/.hbase-snapshot/.tmp/snapshot_1699781514763_ns2_test-16997815020111 2023-11-12T09:31:57,393 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:31:57,394 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:31:57,394 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:31:57,395 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:31:58,471 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop-8312576655057923428.jar 2023-11-12T09:31:58,472 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:31:58,473 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:31:58,473 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:31:58,474 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:31:58,474 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:31:58,475 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:31:58,475 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:31:58,475 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:31:58,476 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:31:58,477 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:31:58,477 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:31:58,478 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:31:58,478 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:31:58,479 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:31:58,479 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:31:58,480 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:31:58,480 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:31:58,481 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:31:58,481 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:31:58,482 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:31:58,482 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:31:58,486 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:31:58,487 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:31:58,487 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:31:58,487 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:31:58,488 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:31:58,488 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:31:58,489 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:31:58,553 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:31:58,583 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781514763_ns2_test-16997815020111' hfile list 2023-11-12T09:31:58,594 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-12T09:31:58,830 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781518773/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:31:58,831 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781518773/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:31:58,986 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:31:59,017 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/f/64a8b4f728734a48b016e4bd27b30a70 output=hdfs://localhost:38513/backupUT/backup_1699781507788/ns2/test-16997815020111/archive/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/f/64a8b4f728734a48b016e4bd27b30a70 2023-11-12T09:31:59,017 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-11-12T09:31:59,889 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:31:59,890 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:31:59,897 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781514763_ns2_test-16997815020111 2023-11-12T09:31:59,897 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781514763_ns2_test-16997815020111 finished. 2023-11-12T09:31:59,897 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781507788_test-1699781502011 2023-11-12T09:31:59,898 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781512635_default_test-1699781502011 to hdfs://localhost:38513/backupUT/backup_1699781507788/default/test-1699781502011/ 2023-11-12T09:31:59,898 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:31:59,924 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:38513, inputRoot=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f 2023-11-12T09:31:59,924 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,35261,1699781489344_-14641243_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38513/backupUT/backup_1699781507788/default/test-1699781502011, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38513/backupUT/backup_1699781507788/default/test-1699781502011/.hbase-snapshot/.tmp/snapshot_1699781512635_default_test-1699781502011 2023-11-12T09:31:59,924 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:38513, inputRoot=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f, snapshotDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_1699781512635_default_test-1699781502011. 2023-11-12T09:31:59,934 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_1699781512635_default_test-1699781502011 to hdfs://localhost:38513/backupUT/backup_1699781507788/default/test-1699781502011/.hbase-snapshot/.tmp/snapshot_1699781512635_default_test-1699781502011 2023-11-12T09:32:00,368 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:00,368 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:00,369 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:00,369 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:01,453 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop-5958716834876308202.jar 2023-11-12T09:32:01,453 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:01,454 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:01,454 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:01,455 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:01,455 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:01,456 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:01,456 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:01,457 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:01,457 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:01,458 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:32:01,459 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:32:01,460 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:32:01,460 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:32:01,461 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:32:01,461 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:32:01,462 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:32:01,462 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:32:01,463 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:32:01,463 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:32:01,464 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:32:01,465 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:32:01,466 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:32:01,466 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:32:01,468 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:32:01,468 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:32:01,469 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:32:01,470 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:32:01,471 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:32:01,501 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:32:01,518 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781512635_default_test-1699781502011' hfile list 2023-11-12T09:32:01,521 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-12T09:32:01,605 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781521569/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:32:01,606 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781521569/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:32:01,674 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:32:01,700 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/f/f8fc91fc02ac435eb7492eb4828be543 output=hdfs://localhost:38513/backupUT/backup_1699781507788/default/test-1699781502011/archive/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/f/f8fc91fc02ac435eb7492eb4828be543 2023-11-12T09:32:01,701 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-11-12T09:32:02,656 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:32:02,657 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:32:02,664 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781512635_default_test-1699781502011 2023-11-12T09:32:02,665 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781512635_default_test-1699781502011 finished. 2023-11-12T09:32:02,665 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1034): ns2:test-16997815020111 2023-11-12T09:32:02,665 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1034): test-1699781502011 2023-11-12T09:32:02,738 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781507788 2023-11-12T09:32:02,738 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:32:03,152 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781507788/.backup.manifest 2023-11-12T09:32:03,152 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781507788 2023-11-12T09:32:03,152 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:32:03,159 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781507788/.backup.manifest 2023-11-12T09:32:03,159 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(358): Backup backup_1699781507788 finished: type=FULL,tablelist=ns2:test-16997815020111;test-1699781502011,targetRootDir=hdfs://localhost:38513/backupUT,startts=1699781512373,completets=1699781522735,bytescopied=0 2023-11-12T09:32:03,159 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:32:03,159 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781514763_ns2_test-16997815020111 2023-11-12T09:32:03,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781514763_ns2_test-16997815020111" 2023-11-12T09:32:03,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781514763_ns2_test-16997815020111 2023-11-12T09:32:03,175 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781514763_ns2_test-16997815020111 for backup backup_1699781507788 succeeded. 2023-11-12T09:32:03,175 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781512635_default_test-1699781502011 2023-11-12T09:32:03,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781512635_default_test-1699781502011" 2023-11-12T09:32:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781512635_default_test-1699781502011 2023-11-12T09:32:03,180 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781512635_default_test-1699781502011 for backup backup_1699781507788 succeeded. 2023-11-12T09:32:03,181 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:32:03,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:32:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:32:03,197 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:32:03,204 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:32:03,222 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(376): Backup backup_1699781507788 completed. 2023-11-12T09:32:03,398 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:32:03,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:32:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781523403 (current time:1699781523403). 2023-11-12T09:32:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:32:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:32:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x660c5280 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@522018f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:03,414 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:60112, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x660c5280 to 127.0.0.1:55425 2023-11-12T09:32:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:32:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:32:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 38 2023-11-12T09:32:03,418 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:32:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:32:03,419 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:32:03,422 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot, current working dir path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:32:03,425 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:32:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:32:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:32:03,832 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:32:03,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb}] 2023-11-12T09:32:03,834 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:03,985 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:03,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37711 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=39 2023-11-12T09:32:03,986 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:32:03,986 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(2819): Flushing 0e93acef4885d53d7a3e4dd8fbdd16bb 2/2 column families, dataSize=1.51 KB heapSize=2.80 KB 2023-11-12T09:32:04,001 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/5ed1e945332b4ef09767e2ffece9f631 is 171, key is trslm:hdfs://localhost:38513/backupUT\x00ns2:test-16997815020111/meta:log-roll-map/1699781522725/Put/seqid=0 2023-11-12T09:32:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:32:04,406 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/5ed1e945332b4ef09767e2ffece9f631 2023-11-12T09:32:04,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/8195231b05ab4f6d978e10371472ca60 is 459, key is session:backup_1699781507788/session:context/1699781523203/Put/seqid=0 2023-11-12T09:32:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:32:04,822 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=758 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/8195231b05ab4f6d978e10371472ca60 2023-11-12T09:32:04,830 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/5ed1e945332b4ef09767e2ffece9f631 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631 2023-11-12T09:32:04,837 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631, entries=6, sequenceid=17, filesize=5.8 K 2023-11-12T09:32:04,838 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/8195231b05ab4f6d978e10371472ca60 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60 2023-11-12T09:32:04,845 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60, entries=2, sequenceid=17, filesize=5.4 K 2023-11-12T09:32:04,846 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(3022): Finished flush of dataSize ~1.51 KB/1543, heapSize ~2.77 KB/2840, currentSize=0 B/0 for 0e93acef4885d53d7a3e4dd8fbdd16bb in 860ms, sequenceid=17, compaction requested=false 2023-11-12T09:32:04,846 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.HRegion(2520): Flush status journal for 0e93acef4885d53d7a3e4dd8fbdd16bb: 2023-11-12T09:32:04,846 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. for snapshot_backup_system completed. 2023-11-12T09:32:04,846 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:32:04,846 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:32:04,846 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631] hfiles 2023-11-12T09:32:04,846 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631 for snapshot=snapshot_backup_system 2023-11-12T09:32:04,846 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60] hfiles 2023-11-12T09:32:04,846 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f for snapshot=snapshot_backup_system 2023-11-12T09:32:04,846 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60 for snapshot=snapshot_backup_system 2023-11-12T09:32:05,254 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:32:05,254 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=39 2023-11-12T09:32:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4026): Remote procedure done, pid=39 2023-11-12T09:32:05,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:05,256 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:05,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=39, resume processing ppid=38 2023-11-12T09:32:05,259 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:32:05,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, ppid=38, state=SUCCESS; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb in 1.4250 sec 2023-11-12T09:32:05,260 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:32:05,261 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:32:05,261 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:32:05,262 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:32:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:32:05,673 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:32:05,681 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:32:05,681 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:32:05,683 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=38, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:32:05,683 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 38 2023-11-12T09:32:05,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=38, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2680 sec 2023-11-12T09:32:07,005 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-12T09:32:07,007 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:34648, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-12T09:32:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:32:07,528 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 38 completed 2023-11-12T09:32:07,528 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(115): Backup backup_1699781523368 started at 1699781527528. 2023-11-12T09:32:07,530 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(119): Backup session backup_1699781523368 has been started. 2023-11-12T09:32:07,535 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1699781502011, ns2:test-16997815020111] 2023-11-12T09:32:07,539 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(76): StartCode 1699781494554for backupID backup_1699781523368 2023-11-12T09:32:07,539 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-11-12T09:32:07,540 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:32:07,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:32:07,541 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:32:07,541 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:32:07,542 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:32:07,542 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:32:07,543 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:07,543 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:07,544 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,544 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:32:07,544 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:32:07,544 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:32:07,545 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,545 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:32:07,545 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:07,545 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:07,546 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:32:07,546 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:07,546 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:32:07,546 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:32:07,546 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:32:07,546 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:32:07,546 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:32:07,547 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:32:07,547 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:32:07,547 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:32:07,551 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,551 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:07,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:07,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:07,551 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:32:07,551 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:32:07,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:07,552 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:07,552 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:07,553 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,553 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:07,553 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:32:07,553 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3f247cb0[Count = 0] remaining members to acquire global barrier 2023-11-12T09:32:07,553 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:32:07,553 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:07,555 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:07,555 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:07,555 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:07,555 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:32:07,555 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,555 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:32:07,555 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:32:07,555 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,555 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781512433 highest: 1699781512433 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,556 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776:(num 1699781512433) roll requested 2023-11-12T09:32:07,568 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781527556, exclude list is [], retry=0 2023-11-12T09:32:07,573 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:32:07,575 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781512433 with entries=113, filesize=13.97 KB; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781527556 2023-11-12T09:32:07,575 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:32:07,575 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781512433 is not closed yet, will try archiving it next time 2023-11-12T09:32:07,575 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta:.meta(num 1699781512457) roll requested 2023-11-12T09:32:07,588 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781527575.meta, exclude list is [], retry=0 2023-11-12T09:32:07,592 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:32:07,594 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781512457.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781527575.meta 2023-11-12T09:32:07,594 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:32:07,594 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781512457.meta is not closed yet, will try archiving it next time 2023-11-12T09:32:07,596 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-11-12T09:32:07,596 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781527556 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,597 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781512457.meta not finished, retry = 0 2023-11-12T09:32:07,608 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:32:07,608 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37711,1699781492776' in zk 2023-11-12T09:32:07,614 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:32:07,614 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,614 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:32:07,614 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:32:07,615 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,615 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:07,615 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:07,616 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:07,616 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:07,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:07,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:07,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:07,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,619 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37711,1699781492776': 2023-11-12T09:32:07,619 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:32:07,619 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:32:07,619 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:32:07,619 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:32:07,619 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:32:07,619 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:32:07,621 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:07,621 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:07,622 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:32:07,622 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:07,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:07,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:07,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:07,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:07,622 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:32:07,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:32:07,622 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:07,623 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:07,623 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,623 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:07,624 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:07,624 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:07,625 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,625 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,625 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:07,626 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:07,626 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,631 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,631 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:07,631 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:32:07,631 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:07,631 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:07,631 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:07,631 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:32:07,631 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:07,631 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:32:07,631 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:07,631 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:32:07,631 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:32:07,631 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-11-12T09:32:07,631 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:32:07,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:32:07,632 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-12T09:32:07,632 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:32:07,632 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:32:07,632 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:32:07,699 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781512457.meta to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781512457.meta 2023-11-12T09:32:07,733 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:32:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:32:07,736 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:37711=1699781494554} newestTimestamps: {jenkins-hbase3.apache.org:37711=1699781512433} 2023-11-12T09:32:07,739 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781495617.meta 2023-11-12T09:32:07,739 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781527575.meta 2023-11-12T09:32:07,739 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781512433 2023-11-12T09:32:07,739 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781527556 2023-11-12T09:32:07,740 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781512457.meta 2023-11-12T09:32:08,151 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(147): Attempting to copy table info for:ns2:test-16997815020111 target: hdfs://localhost:38513/backupUT/backup_1699781523368/ns2/test-16997815020111 descriptor: 'ns2:test-16997815020111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:32:08,151 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:32:08,154 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(152): Starting to write region info for table ns2:test-16997815020111 2023-11-12T09:32:08,560 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(159): Finished writing region info for table ns2:test-16997815020111 2023-11-12T09:32:08,972 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(147): Attempting to copy table info for:test-1699781502011 target: hdfs://localhost:38513/backupUT/backup_1699781523368/default/test-1699781502011 descriptor: 'test-1699781502011', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:32:08,972 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:32:08,974 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(152): Starting to write region info for table test-1699781502011 2023-11-12T09:32:09,380 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(159): Finished writing region info for table test-1699781502011 2023-11-12T09:32:09,391 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:38513/backupUT/.tmp/backup_1699781523368 from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781512433 2023-11-12T09:32:09,394 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1a6f6b9c to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:09,400 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3324bb0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:09,402 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:32:09,403 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:34660, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:32:09,419 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:32:09,420 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1699781502011 2023-11-12T09:32:09,424 DEBUG [hconnection-0x143e389b-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:09,426 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38572, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:09,429 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1699781502011: test-1699781502011; 2023-11-12T09:32:09,430 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16997815020111 2023-11-12T09:32:09,432 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16997815020111: ns2:test-16997815020111; 2023-11-12T09:32:09,432 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-11-12T09:32:09,432 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/hbase-staging/partitions_56a62398-af58-475a-8de1-f5c42d4ff0ac 2023-11-12T09:32:09,877 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:09,877 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:09,878 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:09,879 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,866 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop-2593158068634075807.jar 2023-11-12T09:32:10,866 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,867 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,867 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,868 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,868 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,869 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,869 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,869 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,870 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,870 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:32:10,871 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:32:10,872 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:32:10,872 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:32:10,873 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:32:10,873 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:32:10,873 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:32:10,874 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:32:10,874 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:32:10,875 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:32:10,875 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:32:10,876 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:32:10,878 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,878 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,879 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,879 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,879 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,880 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:10,881 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:32:10,881 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1699781502011,ns2:test-16997815020111 2023-11-12T09:32:10,881 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:32:10,881 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1a6f6b9c to 127.0.0.1:55425 2023-11-12T09:32:10,881 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:10,882 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:32:11,859 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop-7043394437652758400.jar 2023-11-12T09:32:11,864 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:32:11,910 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781512433 for WAL files 2023-11-12T09:32:11,912 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781512433 2023-11-12T09:32:11,993 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781531957/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:32:11,993 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781531957/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:32:12,076 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781512433 for hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781512433 (-9223372036854775808:9223372036854775807) length:14316 2023-11-12T09:32:12,093 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:32:12,094 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:32:12,245 DEBUG [pool-205-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x62dee902 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:12,250 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ce6000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:12,254 DEBUG [hconnection-0x459593f2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:12,255 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38576, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:12,257 DEBUG [pool-205-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x62dee902 to 127.0.0.1:55425 2023-11-12T09:32:12,258 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:12,260 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t10] 2023-11-12T09:32:12,260 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:32:12,275 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/backupUT/.tmp/backup_1699781523368/_temporary/0/_temporary/attempt_local1776627630_0003_r_000001_0/default/test-1699781502011/f/3dfa22f3a7a84ff79af2cb75384f4ac0 is 40, key is row-t110/f:q1/1699781523255/Put/seqid=0 2023-11-12T09:32:12,278 WARN [Thread-847 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:37711] Specified favored nodes: [jenkins-hbase3.apache.org:37711] 2023-11-12T09:32:13,031 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:38513/backupUT 2023-11-12T09:32:13,032 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1699781523368 2023-11-12T09:32:13,032 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-11-12T09:32:13,047 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:38513/backupUT/.tmp/backup_1699781523368, hdfs://localhost:38513/backupUT] 2023-11-12T09:32:13,121 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:32:13,723 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-11-12T09:32:13,725 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1699781523368 2023-11-12T09:32:13,725 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 8797 bytes copied." 2023-11-12T09:32:13,725 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local356278120_0004 completed: true true 2023-11-12T09:32:13,732 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=3777148 FILE: Number of bytes written=5071155 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=107128 HDFS: Number of bytes written=2323071 HDFS: Number of read operations=443 HDFS: Number of large read operations=0 HDFS: Number of write operations=188 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1842872320 File Input Format Counters Bytes Read=1478 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=8797 Bytes Expected=8797 Files Copied=2 DIR_COPY=4 2023-11-12T09:32:13,732 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:38513/backupUT/.tmp/backup_1699781523368 to hdfs://localhost:38513/backupUT finished. 2023-11-12T09:32:13,746 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-11-12T09:32:13,746 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781523368 2023-11-12T09:32:13,754 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-11-12T09:32:13,754 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781523368 2023-11-12T09:32:13,755 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-11-12T09:32:14,162 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:14,163 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781523368 2023-11-12T09:32:14,165 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-11-12T09:32:14,166 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781523368 2023-11-12T09:32:14,168 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-11-12T09:32:14,176 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:14,177 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781523368 2023-11-12T09:32:14,181 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 1 ancestors for the current backup. 2023-11-12T09:32:14,192 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:14,192 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(358): Backup backup_1699781523368 finished: type=INCREMENTAL,tablelist=ns2:test-16997815020111;test-1699781502011,targetRootDir=hdfs://localhost:38513/backupUT,startts=1699781527528,completets=1699781533746,bytescopied=0 2023-11-12T09:32:14,192 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:32:14,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:32:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:32:14,209 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:32:14,212 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:32:14,216 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(376): Backup backup_1699781523368 completed. 2023-11-12T09:32:14,359 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:32:14,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:32:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781534363 (current time:1699781534363). 2023-11-12T09:32:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:32:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:32:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1729e813 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:14,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@187b6420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:14,375 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:38578, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1729e813 to 127.0.0.1:55425 2023-11-12T09:32:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:32:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=40, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:32:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 40 2023-11-12T09:32:14,380 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:32:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=40 2023-11-12T09:32:14,381 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:32:14,384 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot, current working dir path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:32:14,386 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:32:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=40 2023-11-12T09:32:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=40 2023-11-12T09:32:14,793 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:32:14,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb}] 2023-11-12T09:32:14,795 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:14,946 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:14,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37711 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=41 2023-11-12T09:32:14,947 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:32:14,948 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(2819): Flushing 0e93acef4885d53d7a3e4dd8fbdd16bb 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-11-12T09:32:14,952 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/28ae5f7615ab48e09813a26c19b80a8f is 171, key is trslm:hdfs://localhost:38513/backupUT\x00ns2:test-16997815020111/meta:log-roll-map/1699781533737/Put/seqid=0 2023-11-12T09:32:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=40 2023-11-12T09:32:15,357 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/28ae5f7615ab48e09813a26c19b80a8f 2023-11-12T09:32:15,368 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/50a204075f864a39805ebe7f35ecdb42 is 360, key is session:backup_1699781523368/session:context/1699781534210/Put/seqid=0 2023-11-12T09:32:15,373 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/50a204075f864a39805ebe7f35ecdb42 2023-11-12T09:32:15,381 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/28ae5f7615ab48e09813a26c19b80a8f as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f 2023-11-12T09:32:15,389 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f, entries=4, sequenceid=28, filesize=5.7 K 2023-11-12T09:32:15,390 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/50a204075f864a39805ebe7f35ecdb42 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42 2023-11-12T09:32:15,397 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42, entries=2, sequenceid=28, filesize=5.3 K 2023-11-12T09:32:15,398 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for 0e93acef4885d53d7a3e4dd8fbdd16bb in 451ms, sequenceid=28, compaction requested=true 2023-11-12T09:32:15,398 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.HRegion(2520): Flush status journal for 0e93acef4885d53d7a3e4dd8fbdd16bb: 2023-11-12T09:32:15,398 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. for snapshot_backup_system completed. 2023-11-12T09:32:15,398 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:32:15,398 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:32:15,398 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f] hfiles 2023-11-12T09:32:15,398 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631 for snapshot=snapshot_backup_system 2023-11-12T09:32:15,398 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f for snapshot=snapshot_backup_system 2023-11-12T09:32:15,398 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42] hfiles 2023-11-12T09:32:15,398 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f for snapshot=snapshot_backup_system 2023-11-12T09:32:15,398 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60 for snapshot=snapshot_backup_system 2023-11-12T09:32:15,398 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42 for snapshot=snapshot_backup_system 2023-11-12T09:32:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=40 2023-11-12T09:32:15,804 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:32:15,804 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=41 2023-11-12T09:32:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4026): Remote procedure done, pid=41 2023-11-12T09:32:15,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:15,806 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=41, ppid=40, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:15,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-11-12T09:32:15,808 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:32:15,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb in 1.0140 sec 2023-11-12T09:32:15,809 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:32:15,810 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:32:15,810 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:32:15,811 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:32:16,222 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:32:16,231 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:32:16,232 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:32:16,233 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=40, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:32:16,233 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 40 2023-11-12T09:32:16,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=40, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8560 sec 2023-11-12T09:32:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=40 2023-11-12T09:32:16,487 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 40 completed 2023-11-12T09:32:16,487 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(115): Backup backup_1699781534329 started at 1699781536487. 2023-11-12T09:32:16,489 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(119): Backup session backup_1699781534329 has been started. 2023-11-12T09:32:16,490 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1699781502011, ns2:test-16997815020111] 2023-11-12T09:32:16,496 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(76): StartCode 1699781512433for backupID backup_1699781534329 2023-11-12T09:32:16,496 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-11-12T09:32:16,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:32:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:32:16,498 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:32:16,498 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:32:16,499 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:32:16,499 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:32:16,499 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:16,499 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:16,501 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,501 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:32:16,501 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:32:16,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:32:16,501 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,501 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:32:16,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:16,502 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:16,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:32:16,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:16,502 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:32:16,502 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:32:16,503 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:32:16,503 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:32:16,503 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:32:16,503 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:32:16,503 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:32:16,503 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:32:16,506 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:16,506 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,506 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,506 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:16,506 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:16,506 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:32:16,506 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:32:16,507 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:16,507 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:16,507 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:16,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:16,511 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:32:16,511 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@25bc5bae[Count = 0] remaining members to acquire global barrier 2023-11-12T09:32:16,511 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:32:16,511 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:16,512 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:16,512 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:16,512 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:16,513 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:32:16,513 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,513 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:32:16,513 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:32:16,513 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,513 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781527556 highest: 1699781527556 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,513 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776:(num 1699781527556) roll requested 2023-11-12T09:32:16,524 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781536513, exclude list is [], retry=0 2023-11-12T09:32:16,528 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:32:16,530 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781527556 with entries=109, filesize=13.33 KB; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781536513 2023-11-12T09:32:16,530 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:32:16,530 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781527556 is not closed yet, will try archiving it next time 2023-11-12T09:32:16,530 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta:.meta(num 1699781527575) roll requested 2023-11-12T09:32:16,542 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781536531.meta, exclude list is [], retry=0 2023-11-12T09:32:16,546 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:32:16,547 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781527575.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781536531.meta 2023-11-12T09:32:16,547 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:32:16,547 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781527575.meta is not closed yet, will try archiving it next time 2023-11-12T09:32:16,549 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781527575.meta not finished, retry = 0 2023-11-12T09:32:16,553 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:32:16,553 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781536513 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,562 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:32:16,562 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37711,1699781492776' in zk 2023-11-12T09:32:16,566 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,566 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:32:16,566 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:32:16,566 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:32:16,566 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,567 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:16,567 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:16,568 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:16,568 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:16,568 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:16,568 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,569 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:16,569 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:16,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,570 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37711,1699781492776': 2023-11-12T09:32:16,570 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:32:16,570 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:32:16,570 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:32:16,570 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:32:16,570 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:32:16,570 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:32:16,573 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:16,573 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:16,573 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:32:16,573 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:16,573 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:16,573 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:16,573 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:16,573 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:16,573 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:32:16,573 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:16,573 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:32:16,574 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,574 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:16,574 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:16,574 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:16,575 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:16,575 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,575 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,575 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:16,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:16,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,580 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,580 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:32:16,580 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:16,580 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:16,580 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:32:16,580 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:16,580 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:32:16,580 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:16,580 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:16,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:32:16,580 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:16,580 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:32:16,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:32:16,580 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:32:16,581 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-12T09:32:16,581 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:32:16,581 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:32:16,581 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:32:16,651 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781527575.meta to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781527575.meta 2023-11-12T09:32:16,682 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:32:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:32:16,685 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:37711=1699781512433} newestTimestamps: {jenkins-hbase3.apache.org:37711=1699781527556} 2023-11-12T09:32:16,686 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781495617.meta 2023-11-12T09:32:16,686 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781536531.meta 2023-11-12T09:32:16,686 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781527556 2023-11-12T09:32:16,686 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781536513 2023-11-12T09:32:16,687 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781512457.meta 2023-11-12T09:32:16,687 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781527575.meta 2023-11-12T09:32:16,957 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-11-12T09:32:16,957 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-11-12T09:32:17,098 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(147): Attempting to copy table info for:ns2:test-16997815020111 target: hdfs://localhost:38513/backupUT/backup_1699781534329/ns2/test-16997815020111 descriptor: 'ns2:test-16997815020111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:32:17,098 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:32:17,100 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(152): Starting to write region info for table ns2:test-16997815020111 2023-11-12T09:32:17,506 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(159): Finished writing region info for table ns2:test-16997815020111 2023-11-12T09:32:17,917 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(147): Attempting to copy table info for:test-1699781502011 target: hdfs://localhost:38513/backupUT/backup_1699781534329/default/test-1699781502011 descriptor: 'test-1699781502011', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:32:17,917 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:32:17,919 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(152): Starting to write region info for table test-1699781502011 2023-11-12T09:32:18,325 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(159): Finished writing region info for table test-1699781502011 2023-11-12T09:32:18,331 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:38513/backupUT/.tmp/backup_1699781534329 from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781527556 2023-11-12T09:32:18,332 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2442c61e to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:18,338 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ebfc35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:18,339 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:32:18,341 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:58006, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:32:18,344 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:32:18,344 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1699781502011 2023-11-12T09:32:18,349 DEBUG [hconnection-0x5f45d244-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:18,350 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47668, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:18,351 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1699781502011: test-1699781502011; 2023-11-12T09:32:18,352 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16997815020111 2023-11-12T09:32:18,354 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16997815020111: ns2:test-16997815020111; 2023-11-12T09:32:18,354 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-11-12T09:32:18,354 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/hbase-staging/partitions_fce789ed-60b0-40fd-9f89-e34d61e7bda9 2023-11-12T09:32:18,761 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:18,762 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:18,762 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:18,762 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,742 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop-2793635514979008650.jar 2023-11-12T09:32:19,742 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,743 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,743 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,744 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,744 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,744 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,745 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,745 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,745 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,746 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:32:19,747 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:32:19,747 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:32:19,748 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:32:19,748 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:32:19,748 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:32:19,749 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:32:19,749 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:32:19,750 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:32:19,750 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:32:19,750 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:32:19,751 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:32:19,751 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,752 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,752 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,753 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,753 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,753 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:19,754 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:32:19,754 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1699781502011,ns2:test-16997815020111 2023-11-12T09:32:19,754 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:32:19,755 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2442c61e to 127.0.0.1:55425 2023-11-12T09:32:19,755 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:19,756 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:32:20,790 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop-2747270067488110863.jar 2023-11-12T09:32:20,797 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:32:20,814 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781527556 for WAL files 2023-11-12T09:32:20,815 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781527556 2023-11-12T09:32:20,901 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781540860/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:32:20,901 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781540860/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:32:20,968 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781527556 for hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781527556 (-9223372036854775808:9223372036854775807) length:13653 2023-11-12T09:32:20,974 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:32:20,975 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:32:20,996 DEBUG [pool-218-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0db38f90 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:21,003 DEBUG [pool-218-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c6aee9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:21,005 DEBUG [hconnection-0x27d1203a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:21,008 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47670, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:21,010 DEBUG [pool-218-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0db38f90 to 127.0.0.1:55425 2023-11-12T09:32:21,010 DEBUG [pool-218-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:21,012 DEBUG [pool-218-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-11-12T09:32:21,012 DEBUG [pool-218-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:32:21,024 DEBUG [pool-218-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/backupUT/.tmp/backup_1699781534329/_temporary/0/_temporary/attempt_local2128943317_0005_r_000000_0/ns2/test-16997815020111/f/10d914eb7f2f41629149ec6c1ec23c30 is 40, key is row-t210/f:q1/1699781534244/Put/seqid=0 2023-11-12T09:32:21,026 WARN [Thread-972 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:37711] Specified favored nodes: [jenkins-hbase3.apache.org:37711] 2023-11-12T09:32:21,956 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:38513/backupUT 2023-11-12T09:32:21,956 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1699781534329 2023-11-12T09:32:21,956 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-11-12T09:32:21,961 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:38513/backupUT/.tmp/backup_1699781534329, hdfs://localhost:38513/backupUT] 2023-11-12T09:32:21,987 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:32:22,566 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-11-12T09:32:22,568 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1699781534329 2023-11-12T09:32:22,568 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 9114 bytes copied." 2023-11-12T09:32:22,568 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1425776253_0006 completed: true true 2023-11-12T09:32:22,572 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=4413872 FILE: Number of bytes written=7480588 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=153821 HDFS: Number of bytes written=2356044 HDFS: Number of read operations=564 HDFS: Number of large read operations=0 HDFS: Number of write operations=230 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=343 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1842872320 File Input Format Counters Bytes Read=1452 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=9114 Bytes Expected=9114 Files Copied=2 DIR_COPY=4 2023-11-12T09:32:22,572 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:38513/backupUT/.tmp/backup_1699781534329 to hdfs://localhost:38513/backupUT finished. 2023-11-12T09:32:22,582 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-11-12T09:32:22,582 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781534329 2023-11-12T09:32:22,587 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:22,587 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:22,590 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:22,590 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:22,590 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-11-12T09:32:22,590 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781534329 2023-11-12T09:32:22,592 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:22,592 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:22,595 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:22,595 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:22,595 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-11-12T09:32:23,001 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:32:23,001 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781534329 2023-11-12T09:32:23,004 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:23,004 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:23,007 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:23,007 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:23,007 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-11-12T09:32:23,007 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781534329 2023-11-12T09:32:23,010 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:23,010 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:23,012 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:23,013 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:23,013 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-11-12T09:32:23,420 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:32:23,420 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781534329 2023-11-12T09:32:23,423 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:23,423 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:23,426 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:23,426 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:23,426 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 2 ancestors for the current backup. 2023-11-12T09:32:23,832 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:32:23,832 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(358): Backup backup_1699781534329 finished: type=INCREMENTAL,tablelist=ns2:test-16997815020111;test-1699781502011,targetRootDir=hdfs://localhost:38513/backupUT,startts=1699781536487,completets=1699781542582,bytescopied=0 2023-11-12T09:32:23,832 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:32:23,837 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:32:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:32:23,840 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:32:23,842 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:32:23,845 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(376): Backup backup_1699781534329 completed. 2023-11-12T09:32:23,964 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:32:23,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:32:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781543968 (current time:1699781543968). 2023-11-12T09:32:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:32:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:32:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1a900c2f to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103da283, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:23,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:23,979 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47672, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1a900c2f to 127.0.0.1:55425 2023-11-12T09:32:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:32:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:32:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-11-12T09:32:23,983 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:32:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-12T09:32:23,984 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:32:23,986 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot, current working dir path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:32:23,988 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:32:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-12T09:32:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-12T09:32:24,394 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:32:24,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb}] 2023-11-12T09:32:24,397 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:24,548 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:24,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37711 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=43 2023-11-12T09:32:24,549 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:32:24,549 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2819): Flushing 0e93acef4885d53d7a3e4dd8fbdd16bb 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-11-12T09:32:24,553 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/e99f19007bb0490ea42ad6f54a6a2f02 is 171, key is trslm:hdfs://localhost:38513/backupUT\x00ns2:test-16997815020111/meta:log-roll-map/1699781542576/Put/seqid=0 2023-11-12T09:32:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-12T09:32:24,958 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/e99f19007bb0490ea42ad6f54a6a2f02 2023-11-12T09:32:24,967 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/7ae25ee1260b436790453308710d0623 is 360, key is session:backup_1699781534329/session:context/1699781543841/Put/seqid=0 2023-11-12T09:32:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-12T09:32:25,372 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/7ae25ee1260b436790453308710d0623 2023-11-12T09:32:25,380 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/e99f19007bb0490ea42ad6f54a6a2f02 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/e99f19007bb0490ea42ad6f54a6a2f02 2023-11-12T09:32:25,386 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/e99f19007bb0490ea42ad6f54a6a2f02, entries=4, sequenceid=39, filesize=5.7 K 2023-11-12T09:32:25,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/7ae25ee1260b436790453308710d0623 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/7ae25ee1260b436790453308710d0623 2023-11-12T09:32:25,395 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/7ae25ee1260b436790453308710d0623, entries=2, sequenceid=39, filesize=5.3 K 2023-11-12T09:32:25,396 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for 0e93acef4885d53d7a3e4dd8fbdd16bb in 847ms, sequenceid=39, compaction requested=true 2023-11-12T09:32:25,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2520): Flush status journal for 0e93acef4885d53d7a3e4dd8fbdd16bb: 2023-11-12T09:32:25,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. for snapshot_backup_system completed. 2023-11-12T09:32:25,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:32:25,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:32:25,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/e99f19007bb0490ea42ad6f54a6a2f02] hfiles 2023-11-12T09:32:25,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631 for snapshot=snapshot_backup_system 2023-11-12T09:32:25,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f for snapshot=snapshot_backup_system 2023-11-12T09:32:25,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/e99f19007bb0490ea42ad6f54a6a2f02 for snapshot=snapshot_backup_system 2023-11-12T09:32:25,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/7ae25ee1260b436790453308710d0623] hfiles 2023-11-12T09:32:25,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f for snapshot=snapshot_backup_system 2023-11-12T09:32:25,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60 for snapshot=snapshot_backup_system 2023-11-12T09:32:25,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42 for snapshot=snapshot_backup_system 2023-11-12T09:32:25,396 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/7ae25ee1260b436790453308710d0623 for snapshot=snapshot_backup_system 2023-11-12T09:32:25,403 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:32:25,403 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2023-11-12T09:32:25,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4026): Remote procedure done, pid=43 2023-11-12T09:32:25,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:25,404 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:25,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-11-12T09:32:25,407 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:32:25,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb in 1.0100 sec 2023-11-12T09:32:25,408 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:32:25,408 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:32:25,409 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:32:25,409 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:32:25,821 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:32:25,831 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:32:25,831 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:32:25,832 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:32:25,832 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-11-12T09:32:25,833 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8520 sec 2023-11-12T09:32:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-12T09:32:26,090 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 42 completed 2023-11-12T09:32:26,090 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(115): Backup backup_1699781543940 started at 1699781546090. 2023-11-12T09:32:26,091 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(119): Backup session backup_1699781543940 has been started. 2023-11-12T09:32:26,093 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1699781502011, ns2:test-16997815020111] 2023-11-12T09:32:26,100 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(76): StartCode 1699781527556for backupID backup_1699781543940 2023-11-12T09:32:26,100 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-11-12T09:32:26,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:32:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-11-12T09:32:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:32:26,101 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:32:26,101 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:32:26,102 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:32:26,102 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:32:26,103 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:26,103 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:26,104 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:32:26,104 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,105 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:32:26,105 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:32:26,105 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,105 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:32:26,105 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:26,105 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:26,106 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:32:26,106 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:26,106 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:32:26,106 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:32:26,106 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-11-12T09:32:26,106 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:32:26,106 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:32:26,106 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:32:26,107 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:32:26,107 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:32:26,107 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:32:26,109 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,109 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:26,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:26,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:26,110 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:32:26,110 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:32:26,110 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:26,111 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:26,111 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:26,111 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,112 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:26,112 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:32:26,112 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@66484614[Count = 0] remaining members to acquire global barrier 2023-11-12T09:32:26,112 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:32:26,112 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:26,113 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:26,114 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:26,114 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:26,114 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:32:26,114 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:32:26,114 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,114 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:32:26,114 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,114 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781536513 highest: 1699781536513 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,114 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776:(num 1699781536513) roll requested 2023-11-12T09:32:26,125 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781546114, exclude list is [], retry=0 2023-11-12T09:32:26,130 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:32:26,131 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781536513 with entries=109, filesize=13.07 KB; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781546114 2023-11-12T09:32:26,131 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:32:26,132 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781536513 is not closed yet, will try archiving it next time 2023-11-12T09:32:26,132 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta:.meta(num 1699781536531) roll requested 2023-11-12T09:32:26,144 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781546132.meta, exclude list is [], retry=0 2023-11-12T09:32:26,148 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:32:26,150 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781536531.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781546132.meta 2023-11-12T09:32:26,150 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:32:26,150 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781536531.meta is not closed yet, will try archiving it next time 2023-11-12T09:32:26,152 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781536531.meta to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781536531.meta 2023-11-12T09:32:26,154 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:32:26,154 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781546114 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,164 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:32:26,164 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37711,1699781492776' in zk 2023-11-12T09:32:26,170 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,170 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:32:26,170 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:32:26,170 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:32:26,170 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:26,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:26,171 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:26,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:26,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:26,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,173 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:26,173 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:26,174 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,174 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37711,1699781492776': 2023-11-12T09:32:26,175 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:32:26,175 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:32:26,175 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:32:26,175 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:32:26,175 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:32:26,175 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:32:26,177 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:26,177 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:26,177 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:32:26,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:26,177 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:26,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:26,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:26,177 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:26,178 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:32:26,178 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:26,178 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:32:26,178 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,178 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:26,178 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:26,178 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:26,179 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:26,179 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,179 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,179 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:26,180 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:26,180 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,185 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,185 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:26,185 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:26,185 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:32:26,185 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:26,185 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:32:26,185 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:26,185 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:32:26,185 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:26,185 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:32:26,185 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:26,185 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:32:26,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:32:26,185 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-11-12T09:32:26,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:32:26,186 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:32:26,186 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:32:26,186 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-12T09:32:26,186 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:32:26,286 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:32:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:32:26,290 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:37711=1699781527556} newestTimestamps: {jenkins-hbase3.apache.org:37711=1699781536513} 2023-11-12T09:32:26,291 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781495617.meta 2023-11-12T09:32:26,291 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781546132.meta 2023-11-12T09:32:26,291 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781536513 2023-11-12T09:32:26,292 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781546114 2023-11-12T09:32:26,292 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781512457.meta 2023-11-12T09:32:26,292 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781527575.meta 2023-11-12T09:32:26,292 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781536531.meta 2023-11-12T09:32:26,703 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(147): Attempting to copy table info for:ns2:test-16997815020111 target: hdfs://localhost:38513/backupUT/backup_1699781543940/ns2/test-16997815020111 descriptor: 'ns2:test-16997815020111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:32:26,703 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:32:26,705 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(152): Starting to write region info for table ns2:test-16997815020111 2023-11-12T09:32:27,111 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(159): Finished writing region info for table ns2:test-16997815020111 2023-11-12T09:32:27,522 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(147): Attempting to copy table info for:test-1699781502011 target: hdfs://localhost:38513/backupUT/backup_1699781543940/default/test-1699781502011 descriptor: 'test-1699781502011', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:32:27,522 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:32:27,524 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(152): Starting to write region info for table test-1699781502011 2023-11-12T09:32:27,929 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(159): Finished writing region info for table test-1699781502011 2023-11-12T09:32:27,934 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:38513/backupUT/.tmp/backup_1699781543940 from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781536513 2023-11-12T09:32:27,936 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5665f63f to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:27,941 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a4b562c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:27,942 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:32:27,944 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:48582, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:32:27,946 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:32:27,946 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1699781502011 2023-11-12T09:32:27,949 DEBUG [hconnection-0x7dfc9875-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:27,951 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:33814, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:27,952 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1699781502011: test-1699781502011; 2023-11-12T09:32:27,952 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16997815020111 2023-11-12T09:32:27,954 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16997815020111: ns2:test-16997815020111; 2023-11-12T09:32:27,954 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-11-12T09:32:27,954 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/hbase-staging/partitions_40467818-6200-44f8-8ad5-32aeca9e6478 2023-11-12T09:32:28,363 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:28,364 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:28,364 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:28,365 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,322 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop-6541374832332833442.jar 2023-11-12T09:32:29,322 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,323 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,323 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,324 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,324 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,324 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,325 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,325 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,326 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,326 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:32:29,327 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:32:29,327 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:32:29,328 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:32:29,328 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:32:29,329 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:32:29,329 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:32:29,329 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:32:29,330 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:32:29,330 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:32:29,331 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:32:29,331 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:32:29,331 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,332 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,332 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,333 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,333 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,333 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:29,334 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:32:29,334 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1699781502011,ns2:test-16997815020111 2023-11-12T09:32:29,334 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:32:29,334 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5665f63f to 127.0.0.1:55425 2023-11-12T09:32:29,335 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:29,335 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:32:30,330 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop-5265817724111840804.jar 2023-11-12T09:32:30,336 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:32:30,351 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781536513 for WAL files 2023-11-12T09:32:30,352 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781536513 2023-11-12T09:32:30,422 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781550388/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:32:30,423 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781550388/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:32:30,478 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781536513 for hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781536513 (-9223372036854775808:9223372036854775807) length:13396 2023-11-12T09:32:30,484 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:32:30,485 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:32:30,509 DEBUG [pool-231-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x624994bd to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:30,515 DEBUG [pool-231-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55bb86f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:30,517 DEBUG [hconnection-0x3c7ef916-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:30,519 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:33826, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:30,521 DEBUG [pool-231-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x624994bd to 127.0.0.1:55425 2023-11-12T09:32:30,521 DEBUG [pool-231-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:30,522 DEBUG [pool-231-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t1100] 2023-11-12T09:32:30,522 DEBUG [pool-231-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:32:30,532 DEBUG [pool-231-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/backupUT/.tmp/backup_1699781543940/_temporary/0/_temporary/attempt_local133375455_0007_r_000001_0/default/test-1699781502011/f/0c6e603dac8e435bb23b080cff947905 is 42, key is row-t1100/f:q1/1699781543858/Put/seqid=0 2023-11-12T09:32:30,534 WARN [Thread-1107 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:37711] Specified favored nodes: [jenkins-hbase3.apache.org:37711] 2023-11-12T09:32:31,456 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:38513/backupUT 2023-11-12T09:32:31,456 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1699781543940 2023-11-12T09:32:31,456 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-11-12T09:32:31,462 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:38513/backupUT/.tmp/backup_1699781543940, hdfs://localhost:38513/backupUT] 2023-11-12T09:32:31,486 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:32:32,063 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-11-12T09:32:32,065 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1699781543940 2023-11-12T09:32:32,065 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 9085 bytes copied." 2023-11-12T09:32:32,065 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local425311121_0008 completed: true true 2023-11-12T09:32:32,068 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=5054494 FILE: Number of bytes written=9883899 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=202337 HDFS: Number of bytes written=2391182 HDFS: Number of read operations=697 HDFS: Number of large read operations=0 HDFS: Number of write operations=272 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=343 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1829240832 File Input Format Counters Bytes Read=1478 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=9085 Bytes Expected=9085 Files Copied=2 DIR_COPY=4 2023-11-12T09:32:32,068 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:38513/backupUT/.tmp/backup_1699781543940 to hdfs://localhost:38513/backupUT finished. 2023-11-12T09:32:32,079 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-11-12T09:32:32,079 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781543940 2023-11-12T09:32:32,083 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:32:32,084 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:32:32,087 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:32:32,087 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:32,087 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:32,087 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:32,089 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:32,089 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:32,089 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-11-12T09:32:32,089 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781543940 2023-11-12T09:32:32,091 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:32:32,091 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:32:32,093 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:32:32,093 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:32,093 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:32,093 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:32,095 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:32,095 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:32,095 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-11-12T09:32:32,501 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:32:32,501 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781543940 2023-11-12T09:32:32,504 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:32:32,504 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:32:32,507 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:32:32,507 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:32,507 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:32,507 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:32,509 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:32,509 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:32,509 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-11-12T09:32:32,509 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781543940 2023-11-12T09:32:32,511 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:32:32,512 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:32:32,514 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:32:32,514 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:32,514 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:32,514 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:32,516 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:32,516 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:32,516 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-11-12T09:32:32,922 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:32:32,923 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781543940 2023-11-12T09:32:32,925 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:32:32,925 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:32:32,928 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:32:32,928 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:32,928 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:32,928 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:32,930 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:32,930 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:32,930 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 3 ancestors for the current backup. 2023-11-12T09:32:33,336 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:32:33,336 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(358): Backup backup_1699781543940 finished: type=INCREMENTAL,tablelist=ns2:test-16997815020111;test-1699781502011,targetRootDir=hdfs://localhost:38513/backupUT,startts=1699781546090,completets=1699781552079,bytescopied=0 2023-11-12T09:32:33,336 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:32:33,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:32:33,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:32:33,347 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:32:33,348 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:32:33,351 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(376): Backup backup_1699781543940 completed. 2023-11-12T09:32:33,471 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:32:33,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:32:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781553474 (current time:1699781553474). 2023-11-12T09:32:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:32:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:32:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x00a6aabf to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f79d8ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:33,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:33,484 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:33834, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x00a6aabf to 127.0.0.1:55425 2023-11-12T09:32:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:33,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:32:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=44, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:32:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 44 2023-11-12T09:32:33,488 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:32:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-12T09:32:33,489 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:32:33,491 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot, current working dir path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:32:33,493 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:32:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-12T09:32:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-12T09:32:33,901 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:32:33,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb}] 2023-11-12T09:32:33,903 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:34,054 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:34,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37711 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=45 2023-11-12T09:32:34,055 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:32:34,055 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(2819): Flushing 0e93acef4885d53d7a3e4dd8fbdd16bb 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-11-12T09:32:34,059 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/9c02b6ae5d514a4b9d2b2ffef3fc5209 is 171, key is trslm:hdfs://localhost:38513/backupUT\x00ns2:test-16997815020111/meta:log-roll-map/1699781552073/Put/seqid=0 2023-11-12T09:32:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-12T09:32:34,412 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2023-11-12T09:32:34,465 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/9c02b6ae5d514a4b9d2b2ffef3fc5209 2023-11-12T09:32:34,475 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/48d5803e75234c9f8c50d40123730dbe is 360, key is session:backup_1699781543940/session:context/1699781553348/Put/seqid=0 2023-11-12T09:32:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-12T09:32:34,881 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/48d5803e75234c9f8c50d40123730dbe 2023-11-12T09:32:34,888 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/9c02b6ae5d514a4b9d2b2ffef3fc5209 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/9c02b6ae5d514a4b9d2b2ffef3fc5209 2023-11-12T09:32:34,893 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/9c02b6ae5d514a4b9d2b2ffef3fc5209, entries=4, sequenceid=50, filesize=5.7 K 2023-11-12T09:32:34,894 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/48d5803e75234c9f8c50d40123730dbe as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/48d5803e75234c9f8c50d40123730dbe 2023-11-12T09:32:34,900 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/48d5803e75234c9f8c50d40123730dbe, entries=2, sequenceid=50, filesize=5.3 K 2023-11-12T09:32:34,900 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for 0e93acef4885d53d7a3e4dd8fbdd16bb in 845ms, sequenceid=50, compaction requested=true 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.HRegion(2520): Flush status journal for 0e93acef4885d53d7a3e4dd8fbdd16bb: 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. for snapshot_backup_system completed. 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/e99f19007bb0490ea42ad6f54a6a2f02, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/9c02b6ae5d514a4b9d2b2ffef3fc5209] hfiles 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631 for snapshot=snapshot_backup_system 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f for snapshot=snapshot_backup_system 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/e99f19007bb0490ea42ad6f54a6a2f02 for snapshot=snapshot_backup_system 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/9c02b6ae5d514a4b9d2b2ffef3fc5209 for snapshot=snapshot_backup_system 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/7ae25ee1260b436790453308710d0623, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/48d5803e75234c9f8c50d40123730dbe] hfiles 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f for snapshot=snapshot_backup_system 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60 for snapshot=snapshot_backup_system 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42 for snapshot=snapshot_backup_system 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/7ae25ee1260b436790453308710d0623 for snapshot=snapshot_backup_system 2023-11-12T09:32:34,901 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/48d5803e75234c9f8c50d40123730dbe for snapshot=snapshot_backup_system 2023-11-12T09:32:35,308 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:32:35,308 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=45 2023-11-12T09:32:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4026): Remote procedure done, pid=45 2023-11-12T09:32:35,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:35,310 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:35,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-11-12T09:32:35,313 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:32:35,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb in 1.4100 sec 2023-11-12T09:32:35,314 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:32:35,315 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:32:35,315 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:32:35,316 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:32:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-12T09:32:35,726 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:32:35,736 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:32:35,736 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:32:35,738 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=44, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:32:35,738 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 44 2023-11-12T09:32:35,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=44, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2530 sec 2023-11-12T09:32:36,315 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 2e735f09d7b85fcf9fb65b6e7191fab4 changed from -1.0 to 0.0, refreshing cache 2023-11-12T09:32:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-12T09:32:37,597 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 44 completed 2023-11-12T09:32:37,597 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(115): Backup backup_1699781553447 started at 1699781557597. 2023-11-12T09:32:37,599 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(119): Backup session backup_1699781553447 has been started. 2023-11-12T09:32:37,600 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1699781502011, ns2:test-16997815020111] 2023-11-12T09:32:37,606 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(76): StartCode 1699781536513for backupID backup_1699781553447 2023-11-12T09:32:37,606 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-11-12T09:32:37,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:32:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:32:37,607 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:32:37,607 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:32:37,608 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:32:37,608 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:32:37,609 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:37,609 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:37,613 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,613 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:32:37,613 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:32:37,614 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:32:37,614 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,614 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:32:37,614 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:37,614 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:37,615 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:32:37,615 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:37,615 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:32:37,615 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:32:37,615 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:32:37,615 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:32:37,615 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:32:37,616 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:32:37,616 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:32:37,616 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:32:37,618 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,618 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:37,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:37,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:37,619 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:32:37,619 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:32:37,619 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:37,619 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:37,620 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:37,620 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,620 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:37,620 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:32:37,620 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1644b0a6[Count = 0] remaining members to acquire global barrier 2023-11-12T09:32:37,620 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:32:37,621 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:37,623 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:37,623 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:37,623 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:37,624 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:32:37,624 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,624 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:32:37,624 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:32:37,624 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,624 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781546114 highest: 1699781546114 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,624 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776:(num 1699781546114) roll requested 2023-11-12T09:32:37,635 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781557624, exclude list is [], retry=0 2023-11-12T09:32:37,640 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:32:37,642 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781546114 with entries=109, filesize=13.56 KB; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781557624 2023-11-12T09:32:37,643 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:32:37,643 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781546114 is not closed yet, will try archiving it next time 2023-11-12T09:32:37,643 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta:.meta(num 1699781546132) roll requested 2023-11-12T09:32:37,646 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781546114 not finished, retry = 0 2023-11-12T09:32:37,661 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781557643.meta, exclude list is [], retry=0 2023-11-12T09:32:37,665 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-11-12T09:32:37,665 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781557624 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,665 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:32:37,667 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781546132.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781557643.meta 2023-11-12T09:32:37,667 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:32:37,667 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781546132.meta is not closed yet, will try archiving it next time 2023-11-12T09:32:37,670 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781546132.meta to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781546132.meta 2023-11-12T09:32:37,674 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:32:37,674 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37711,1699781492776' in zk 2023-11-12T09:32:37,677 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:32:37,677 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,677 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:32:37,677 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:32:37,678 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,678 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:37,678 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:37,678 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:37,679 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:37,679 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:37,679 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,680 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:37,680 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:37,680 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,681 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37711,1699781492776': 2023-11-12T09:32:37,681 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:32:37,681 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:32:37,681 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:32:37,681 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:32:37,681 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:32:37,681 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:32:37,683 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:37,683 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:37,683 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:32:37,683 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:37,683 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:37,683 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:37,683 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:37,683 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:37,684 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:32:37,684 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:32:37,684 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:37,684 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,684 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:37,690 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:37,690 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:37,690 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:37,691 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,691 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,691 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:37,692 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:37,692 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,698 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,699 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:32:37,699 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:37,699 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:32:37,699 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:37,699 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:37,699 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:37,699 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:32:37,699 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:32:37,699 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:37,699 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:32:37,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:32:37,699 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:37,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:32:37,700 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-12T09:32:37,700 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:32:37,700 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:32:37,700 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:32:37,800 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:32:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:32:37,804 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:37711=1699781536513} newestTimestamps: {jenkins-hbase3.apache.org:37711=1699781546114} 2023-11-12T09:32:37,806 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781495617.meta 2023-11-12T09:32:37,806 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781557643.meta 2023-11-12T09:32:37,806 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781546114 2023-11-12T09:32:37,806 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781557624 2023-11-12T09:32:37,807 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781512457.meta 2023-11-12T09:32:37,807 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781527575.meta 2023-11-12T09:32:37,807 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781536531.meta 2023-11-12T09:32:37,807 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781546132.meta 2023-11-12T09:32:38,218 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(147): Attempting to copy table info for:ns2:test-16997815020111 target: hdfs://localhost:38513/backupUT/backup_1699781553447/ns2/test-16997815020111 descriptor: 'ns2:test-16997815020111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:32:38,218 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:32:38,220 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(152): Starting to write region info for table ns2:test-16997815020111 2023-11-12T09:32:38,625 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(159): Finished writing region info for table ns2:test-16997815020111 2023-11-12T09:32:39,035 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(147): Attempting to copy table info for:test-1699781502011 target: hdfs://localhost:38513/backupUT/backup_1699781553447/default/test-1699781502011 descriptor: 'test-1699781502011', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:32:39,035 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:32:39,037 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(152): Starting to write region info for table test-1699781502011 2023-11-12T09:32:39,442 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(159): Finished writing region info for table test-1699781502011 2023-11-12T09:32:39,448 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:38513/backupUT/.tmp/backup_1699781553447 from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781546114 2023-11-12T09:32:39,449 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7d1f3e23 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:39,458 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c082159, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:39,460 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:32:39,461 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:33474, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:32:39,462 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:32:39,462 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1699781502011 2023-11-12T09:32:39,465 DEBUG [hconnection-0x6a9bbe7d-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:39,467 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57802, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:39,469 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1699781502011: test-1699781502011; 2023-11-12T09:32:39,469 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16997815020111 2023-11-12T09:32:39,470 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16997815020111: ns2:test-16997815020111; 2023-11-12T09:32:39,470 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-11-12T09:32:39,471 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/hbase-staging/partitions_48ce6fe6-b592-49fa-ac1c-c00b6eb635ba 2023-11-12T09:32:39,879 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:39,879 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:39,880 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:39,880 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,833 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop-8576099794191473305.jar 2023-11-12T09:32:40,833 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,833 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,834 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,834 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,835 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,835 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,835 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,836 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,836 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,837 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:32:40,837 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:32:40,838 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:32:40,838 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:32:40,839 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:32:40,839 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:32:40,839 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:32:40,840 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:32:40,840 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:32:40,840 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:32:40,841 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:32:40,841 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:32:40,842 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,842 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,843 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,843 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,843 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,844 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:40,844 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:32:40,844 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1699781502011,ns2:test-16997815020111 2023-11-12T09:32:40,844 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:32:40,845 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7d1f3e23 to 127.0.0.1:55425 2023-11-12T09:32:40,845 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:40,846 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:32:41,802 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop-7801571411583910366.jar 2023-11-12T09:32:41,808 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:32:41,822 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781546114 for WAL files 2023-11-12T09:32:41,823 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781546114 2023-11-12T09:32:41,893 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781561859/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:32:41,894 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781561859/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:32:41,945 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781546114 for hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781546114 (-9223372036854775808:9223372036854775807) length:13891 2023-11-12T09:32:41,950 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:32:41,950 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:32:41,970 DEBUG [pool-244-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1e6151c9 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:41,975 DEBUG [pool-244-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@250351f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:41,977 DEBUG [hconnection-0x7be42dd7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:41,978 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57804, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:41,979 DEBUG [pool-244-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1e6151c9 to 127.0.0.1:55425 2023-11-12T09:32:41,979 DEBUG [pool-244-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:41,980 DEBUG [pool-244-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t1100] 2023-11-12T09:32:41,980 DEBUG [pool-244-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-11-12T09:32:41,989 DEBUG [pool-244-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/backupUT/.tmp/backup_1699781553447/_temporary/0/_temporary/attempt_local193026642_0009_r_000000_0/ns2/test-16997815020111/f/738dad47d4f047d6a1a782dd9e183c15 is 42, key is row-t1100/f:q1/1699781553364/Put/seqid=0 2023-11-12T09:32:41,990 WARN [Thread-1232 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:37711] Specified favored nodes: [jenkins-hbase3.apache.org:37711] 2023-11-12T09:32:42,926 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:38513/backupUT 2023-11-12T09:32:42,926 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1699781553447 2023-11-12T09:32:42,926 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-11-12T09:32:42,931 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:38513/backupUT/.tmp/backup_1699781553447, hdfs://localhost:38513/backupUT] 2023-11-12T09:32:42,955 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:32:43,533 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-11-12T09:32:43,535 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1699781553447 2023-11-12T09:32:43,535 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 9404 bytes copied." 2023-11-12T09:32:43,535 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local977475030_0010 completed: true true 2023-11-12T09:32:43,538 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=5691866 FILE: Number of bytes written=12288122 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=255940 HDFS: Number of bytes written=2431246 HDFS: Number of read operations=843 HDFS: Number of large read operations=0 HDFS: Number of write operations=315 Map-Reduce Framework Map input records=6 Map output records=0 Input split bytes=343 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1829240832 File Input Format Counters Bytes Read=1452 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=9404 Bytes Expected=9404 Files Copied=2 DIR_COPY=4 2023-11-12T09:32:43,538 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:38513/backupUT/.tmp/backup_1699781553447 to hdfs://localhost:38513/backupUT finished. 2023-11-12T09:32:43,548 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-11-12T09:32:43,548 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781553447 2023-11-12T09:32:43,551 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781543940 to construct the dependency. 2023-11-12T09:32:43,552 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781543940 2023-11-12T09:32:43,554 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:32:43,554 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781543940BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:43,554 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:32:43,554 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:32:43,556 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:32:43,556 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:43,556 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:43,556 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:43,558 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:43,558 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:43,558 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-11-12T09:32:43,558 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781553447 2023-11-12T09:32:43,560 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781543940 to construct the dependency. 2023-11-12T09:32:43,560 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781543940 2023-11-12T09:32:43,562 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:32:43,562 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781543940BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:43,562 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:32:43,562 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:32:43,564 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:32:43,564 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:43,564 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:43,564 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:43,566 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:43,566 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:43,566 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-11-12T09:32:43,972 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781553447/.backup.manifest 2023-11-12T09:32:43,972 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781553447 2023-11-12T09:32:43,974 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781543940 to construct the dependency. 2023-11-12T09:32:43,974 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781543940 2023-11-12T09:32:43,977 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:32:43,977 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781543940BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:43,977 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:32:43,978 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:32:43,980 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:32:43,980 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:43,980 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:43,980 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:43,982 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:43,982 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:43,982 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-11-12T09:32:43,982 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781553447 2023-11-12T09:32:43,984 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781543940 to construct the dependency. 2023-11-12T09:32:43,984 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781543940 2023-11-12T09:32:43,986 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:32:43,986 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781543940BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:43,986 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:32:43,986 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:32:43,988 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:32:43,988 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:43,988 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:43,988 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:43,990 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:43,990 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:43,990 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-11-12T09:32:44,395 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781553447/.backup.manifest 2023-11-12T09:32:44,396 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781553447 2023-11-12T09:32:44,398 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781543940 to construct the dependency. 2023-11-12T09:32:44,398 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781543940 2023-11-12T09:32:44,401 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:32:44,402 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781543940BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:44,402 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:32:44,402 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:32:44,404 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:32:44,404 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:44,404 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:32:44,404 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:32:44,407 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:32:44,407 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:32:44,407 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 4 ancestors for the current backup. 2023-11-12T09:32:44,813 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781553447/.backup.manifest 2023-11-12T09:32:44,813 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(358): Backup backup_1699781553447 finished: type=INCREMENTAL,tablelist=ns2:test-16997815020111;test-1699781502011,targetRootDir=hdfs://localhost:38513/backupUT,startts=1699781557597,completets=1699781563548,bytescopied=0 2023-11-12T09:32:44,813 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:32:44,819 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:32:44,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:32:44,822 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:32:44,824 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:32:44,827 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(376): Backup backup_1699781553447 completed. 2023-11-12T09:32:44,848 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:32:44,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:32:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781564851 (current time:1699781564851). 2023-11-12T09:32:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:32:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:32:44,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x120fe123 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4311b33a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:44,861 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:57820, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:44,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x120fe123 to 127.0.0.1:55425 2023-11-12T09:32:44,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:44,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:32:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=46, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:32:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 46 2023-11-12T09:32:44,865 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:32:44,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2023-11-12T09:32:44,866 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:32:44,868 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot, current working dir path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:32:44,870 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:32:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2023-11-12T09:32:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2023-11-12T09:32:45,277 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:32:45,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb}] 2023-11-12T09:32:45,279 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:45,430 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:45,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37711 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=47 2023-11-12T09:32:45,431 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:32:45,431 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(2819): Flushing 0e93acef4885d53d7a3e4dd8fbdd16bb 2/2 column families, dataSize=1.37 KB heapSize=2.52 KB 2023-11-12T09:32:45,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/18cd51f7497643d49709c1a7bd387f23 is 171, key is trslm:hdfs://localhost:38513/backupUT\x00ns2:test-16997815020111/meta:log-roll-map/1699781563542/Put/seqid=0 2023-11-12T09:32:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2023-11-12T09:32:45,843 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/18cd51f7497643d49709c1a7bd387f23 2023-11-12T09:32:45,852 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/2e78fea284d9486c91abac25af6428e0 is 360, key is session:backup_1699781553447/session:context/1699781564823/Put/seqid=0 2023-11-12T09:32:45,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2023-11-12T09:32:46,258 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=873 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/2e78fea284d9486c91abac25af6428e0 2023-11-12T09:32:46,266 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/18cd51f7497643d49709c1a7bd387f23 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/18cd51f7497643d49709c1a7bd387f23 2023-11-12T09:32:46,272 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/18cd51f7497643d49709c1a7bd387f23, entries=4, sequenceid=61, filesize=5.7 K 2023-11-12T09:32:46,273 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/2e78fea284d9486c91abac25af6428e0 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/2e78fea284d9486c91abac25af6428e0 2023-11-12T09:32:46,279 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/2e78fea284d9486c91abac25af6428e0, entries=2, sequenceid=61, filesize=5.3 K 2023-11-12T09:32:46,280 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(3022): Finished flush of dataSize ~1.37 KB/1404, heapSize ~2.49 KB/2552, currentSize=0 B/0 for 0e93acef4885d53d7a3e4dd8fbdd16bb in 849ms, sequenceid=61, compaction requested=true 2023-11-12T09:32:46,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.HRegion(2520): Flush status journal for 0e93acef4885d53d7a3e4dd8fbdd16bb: 2023-11-12T09:32:46,280 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. for snapshot_backup_system completed. 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/e99f19007bb0490ea42ad6f54a6a2f02, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/9c02b6ae5d514a4b9d2b2ffef3fc5209, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/18cd51f7497643d49709c1a7bd387f23] hfiles 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631 for snapshot=snapshot_backup_system 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f for snapshot=snapshot_backup_system 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/e99f19007bb0490ea42ad6f54a6a2f02 for snapshot=snapshot_backup_system 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/9c02b6ae5d514a4b9d2b2ffef3fc5209 for snapshot=snapshot_backup_system 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/18cd51f7497643d49709c1a7bd387f23 for snapshot=snapshot_backup_system 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/7ae25ee1260b436790453308710d0623, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/48d5803e75234c9f8c50d40123730dbe, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/2e78fea284d9486c91abac25af6428e0] hfiles 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f for snapshot=snapshot_backup_system 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60 for snapshot=snapshot_backup_system 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42 for snapshot=snapshot_backup_system 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/7ae25ee1260b436790453308710d0623 for snapshot=snapshot_backup_system 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/48d5803e75234c9f8c50d40123730dbe for snapshot=snapshot_backup_system 2023-11-12T09:32:46,281 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/2e78fea284d9486c91abac25af6428e0 for snapshot=snapshot_backup_system 2023-11-12T09:32:46,687 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:32:46,687 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=47 2023-11-12T09:32:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4026): Remote procedure done, pid=47 2023-11-12T09:32:46,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:46,689 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:46,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=47, resume processing ppid=46 2023-11-12T09:32:46,690 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:32:46,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, ppid=46, state=SUCCESS; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb in 1.4120 sec 2023-11-12T09:32:46,691 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:32:46,692 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:32:46,692 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:32:46,693 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:32:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2023-11-12T09:32:47,102 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:32:47,114 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:32:47,114 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:32:47,115 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=46, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:32:47,115 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 46 2023-11-12T09:32:47,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=46, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2520 sec 2023-11-12T09:32:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2023-11-12T09:32:48,972 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 46 completed 2023-11-12T09:32:48,972 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(115): Backup backup_1699781564837 started at 1699781568972. 2023-11-12T09:32:48,974 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(119): Backup session backup_1699781564837 has been started. 2023-11-12T09:32:48,979 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:32:48,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:32:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-11-12T09:32:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:32:48,980 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:32:48,980 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:32:48,981 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:32:48,981 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:32:48,982 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:48,982 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:48,983 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:32:48,983 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:48,983 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:32:48,983 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:32:48,984 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:48,984 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:32:48,984 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:48,984 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:48,985 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:32:48,985 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:48,985 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:32:48,985 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:32:48,985 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-11-12T09:32:48,985 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:32:48,985 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:32:48,985 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:32:48,986 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:32:48,986 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:32:48,986 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:32:48,989 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:48,990 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:48,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:48,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:48,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:48,990 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:32:48,990 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:32:48,990 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:48,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:48,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:48,991 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:48,992 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:48,995 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:32:48,995 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@a999f22[Count = 0] remaining members to acquire global barrier 2023-11-12T09:32:48,995 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:32:48,995 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:48,997 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:48,997 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:48,997 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:32:48,998 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:48,998 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:48,998 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:32:48,998 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:32:48,998 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:48,998 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781557624 highest: 1699781557624 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:48,998 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776:(num 1699781557624) roll requested 2023-11-12T09:32:49,010 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781568998, exclude list is [], retry=0 2023-11-12T09:32:49,015 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:32:49,017 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781557624 with entries=10, filesize=2.51 KB; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781568998 2023-11-12T09:32:49,017 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:32:49,017 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781557624 is not closed yet, will try archiving it next time 2023-11-12T09:32:49,017 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta:.meta(num 1699781557643) roll requested 2023-11-12T09:32:49,029 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781569017.meta, exclude list is [], retry=0 2023-11-12T09:32:49,032 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:32:49,034 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781557643.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781569017.meta 2023-11-12T09:32:49,034 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:32:49,034 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781557643.meta is not closed yet, will try archiving it next time 2023-11-12T09:32:49,036 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781557643.meta to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781557643.meta 2023-11-12T09:32:49,038 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:32:49,038 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781568998 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:49,046 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:32:49,046 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37711,1699781492776' in zk 2023-11-12T09:32:49,049 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:49,049 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:32:49,049 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:32:49,049 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:32:49,050 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:49,050 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:49,050 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:49,050 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:49,050 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:49,051 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:49,051 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:49,051 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:49,052 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:49,052 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:49,053 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37711,1699781492776': 2023-11-12T09:32:49,053 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:32:49,053 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:32:49,053 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:32:49,053 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:32:49,053 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:32:49,053 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:32:49,055 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:49,055 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:49,055 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:32:49,055 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:49,055 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:49,055 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:49,055 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:49,055 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:49,055 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:32:49,055 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:49,055 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:32:49,056 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:49,056 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:49,056 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:49,056 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:49,057 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:49,057 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:49,057 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:49,058 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:49,058 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:49,058 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:49,062 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:49,062 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:49,062 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:49,062 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:49,062 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:49,062 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:32:49,062 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:49,062 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:49,062 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:32:49,062 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:32:49,062 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:32:49,062 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:32:49,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:32:49,062 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-11-12T09:32:49,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:32:49,063 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:32:49,063 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:32:49,063 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-12T09:32:49,063 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:32:49,163 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:32:49,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:32:49,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 } 2023-11-12T09:32:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781569168 (current time:1699781569168). 2023-11-12T09:32:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:32:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699781569167_ns3_test-16997815020112 VERSION not specified, setting to 2 2023-11-12T09:32:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x11a29e83 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cc9636d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:49,181 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:33460, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x11a29e83 to 127.0.0.1:55425 2023-11-12T09:32:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:32:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 } 2023-11-12T09:32:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-11-12T09:32:49,185 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:32:49,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-12T09:32:49,186 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:32:49,189 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot, current working dir path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_1699781569167_ns3_test-16997815020112 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:32:49,191 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:32:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-12T09:32:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-12T09:32:49,597 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:32:49,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure 13632229548fde4afa1b285f20ec0adf}] 2023-11-12T09:32:49,599 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure 13632229548fde4afa1b285f20ec0adf 2023-11-12T09:32:49,750 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:49,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37711 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=49 2023-11-12T09:32:49,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:32:49,752 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.HRegion(2520): Flush status journal for 13632229548fde4afa1b285f20ec0adf: 2023-11-12T09:32:49,752 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. for snapshot_1699781569167_ns3_test-16997815020112 completed. 2023-11-12T09:32:49,752 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf.' region-info for snapshot=snapshot_1699781569167_ns3_test-16997815020112 2023-11-12T09:32:49,752 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:32:49,752 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-12T09:32:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-12T09:32:50,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:32:50,158 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=49 2023-11-12T09:32:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4026): Remote procedure done, pid=49 2023-11-12T09:32:50,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699781569167_ns3_test-16997815020112 on region 13632229548fde4afa1b285f20ec0adf 2023-11-12T09:32:50,160 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=49, ppid=48, state=RUNNABLE; SnapshotRegionProcedure 13632229548fde4afa1b285f20ec0adf 2023-11-12T09:32:50,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-11-12T09:32:50,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; SnapshotRegionProcedure 13632229548fde4afa1b285f20ec0adf in 564 msec 2023-11-12T09:32:50,162 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:32:50,163 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:32:50,163 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:32:50,163 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699781569167_ns3_test-16997815020112 2023-11-12T09:32:50,164 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_1699781569167_ns3_test-16997815020112 2023-11-12T09:32:50,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-12T09:32:50,573 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:32:50,578 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:32:50,578 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_1699781569167_ns3_test-16997815020112 to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_1699781569167_ns3_test-16997815020112 2023-11-12T09:32:50,579 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:32:50,580 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-11-12T09:32:50,581 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=snapshot_1699781569167_ns3_test-16997815020112 table=ns3:test-16997815020112 type=FLUSH ttl=0 } in 1.3970 sec 2023-11-12T09:32:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-12T09:32:51,291 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: ns3:test-16997815020112, procId: 48 completed 2023-11-12T09:32:51,291 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699781564837 2023-11-12T09:32:51,291 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:32:51,291 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-12T09:32:51,292 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699781564837_ns3:test-16997815020112 2023-11-12T09:32:51,292 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699781569167_ns3_test-16997815020112 to hdfs://localhost:38513/backupUT/backup_1699781564837/ns3/test-16997815020112/ 2023-11-12T09:32:51,292 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:32:51,316 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:38513, inputRoot=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f 2023-11-12T09:32:51,316 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,35261,1699781489344_1876468073_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:38513/backupUT/backup_1699781564837/ns3/test-16997815020112, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:38513/backupUT/backup_1699781564837/ns3/test-16997815020112/.hbase-snapshot/.tmp/snapshot_1699781569167_ns3_test-16997815020112 2023-11-12T09:32:51,316 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:38513, inputRoot=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f, snapshotDir=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_1699781569167_ns3_test-16997815020112. 2023-11-12T09:32:51,329 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_1699781569167_ns3_test-16997815020112 to hdfs://localhost:38513/backupUT/backup_1699781564837/ns3/test-16997815020112/.hbase-snapshot/.tmp/snapshot_1699781569167_ns3_test-16997815020112 2023-11-12T09:32:51,748 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:51,749 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:51,749 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:51,750 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:52,734 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop-762655792921817828.jar 2023-11-12T09:32:52,734 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:52,735 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:52,735 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:52,736 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:52,736 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:52,736 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:52,737 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:52,737 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:52,737 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:32:52,738 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:32:52,739 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:32:52,739 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:32:52,740 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:32:52,740 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:32:52,741 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:32:52,741 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:32:52,741 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:32:52,742 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:32:52,742 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:32:52,743 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:32:52,743 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:32:52,744 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:32:52,744 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:32:52,745 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:32:52,745 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:32:52,745 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:32:52,746 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:32:52,747 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:32:52,774 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:32:52,793 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699781569167_ns3_test-16997815020112' hfile list 2023-11-12T09:32:52,864 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781572831/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:32:52,864 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781572831/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:32:53,893 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:32:53,894 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:32:53,901 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699781569167_ns3_test-16997815020112 2023-11-12T09:32:53,901 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699781569167_ns3_test-16997815020112 finished. 2023-11-12T09:32:53,901 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1034): ns3:test-16997815020112 2023-11-12T09:32:53,908 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781564837 2023-11-12T09:32:53,908 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:32:54,313 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781564837/.backup.manifest 2023-11-12T09:32:54,313 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(358): Backup backup_1699781564837 finished: type=FULL,tablelist=ns3:test-16997815020112,targetRootDir=hdfs://localhost:38513/backupUT,startts=1699781568972,completets=1699781573908,bytescopied=0 2023-11-12T09:32:54,313 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:32:54,313 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699781569167_ns3_test-16997815020112 2023-11-12T09:32:54,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699781569167_ns3_test-16997815020112" 2023-11-12T09:32:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699781569167_ns3_test-16997815020112 2023-11-12T09:32:54,318 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699781569167_ns3_test-16997815020112 for backup backup_1699781564837 succeeded. 2023-11-12T09:32:54,318 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:32:54,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:32:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:32:54,326 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:32:54,327 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:32:54,334 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(376): Backup backup_1699781564837 completed. 2023-11-12T09:32:54,369 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:32:54,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:32:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781574373 (current time:1699781574373). 2023-11-12T09:32:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:32:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:32:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x54dced0f to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:32:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c0ec89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:32:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:32:54,385 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:33470, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:32:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x54dced0f to 127.0.0.1:55425 2023-11-12T09:32:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:32:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:32:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:32:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-11-12T09:32:54,390 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:32:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:32:54,391 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:32:54,393 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot, current working dir path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:32:54,395 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:32:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:32:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:32:54,802 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:32:54,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb}] 2023-11-12T09:32:54,804 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:54,955 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:54,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37711 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2023-11-12T09:32:54,956 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:32:54,956 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing 0e93acef4885d53d7a3e4dd8fbdd16bb 2/2 column families, dataSize=1.14 KB heapSize=2.23 KB 2023-11-12T09:32:54,959 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/0c4df14ef8d447179ad82847c1682b6f is 171, key is trslm:hdfs://localhost:38513/backupUT\x00ns3:test-16997815020112/meta:log-roll-map/1699781573903/Put/seqid=0 2023-11-12T09:32:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:32:55,364 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=458 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/0c4df14ef8d447179ad82847c1682b6f 2023-11-12T09:32:55,372 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/a18fbc9163324723bd9ebd8c9c7c2829 is 446, key is session:backup_1699781564837/session:context/1699781574326/Put/seqid=0 2023-11-12T09:32:55,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:32:55,777 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=712 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/a18fbc9163324723bd9ebd8c9c7c2829 2023-11-12T09:32:55,783 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/0c4df14ef8d447179ad82847c1682b6f as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/0c4df14ef8d447179ad82847c1682b6f 2023-11-12T09:32:55,788 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/0c4df14ef8d447179ad82847c1682b6f, entries=4, sequenceid=72, filesize=5.6 K 2023-11-12T09:32:55,789 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/a18fbc9163324723bd9ebd8c9c7c2829 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/a18fbc9163324723bd9ebd8c9c7c2829 2023-11-12T09:32:55,794 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/a18fbc9163324723bd9ebd8c9c7c2829, entries=2, sequenceid=72, filesize=5.4 K 2023-11-12T09:32:55,795 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1170, heapSize ~2.20 KB/2248, currentSize=0 B/0 for 0e93acef4885d53d7a3e4dd8fbdd16bb in 839ms, sequenceid=72, compaction requested=true 2023-11-12T09:32:55,795 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for 0e93acef4885d53d7a3e4dd8fbdd16bb: 2023-11-12T09:32:55,795 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. for snapshot_backup_system completed. 2023-11-12T09:32:55,795 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:32:55,795 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:32:55,795 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/e99f19007bb0490ea42ad6f54a6a2f02, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/9c02b6ae5d514a4b9d2b2ffef3fc5209, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/18cd51f7497643d49709c1a7bd387f23, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/0c4df14ef8d447179ad82847c1682b6f] hfiles 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631 for snapshot=snapshot_backup_system 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f for snapshot=snapshot_backup_system 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/e99f19007bb0490ea42ad6f54a6a2f02 for snapshot=snapshot_backup_system 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/9c02b6ae5d514a4b9d2b2ffef3fc5209 for snapshot=snapshot_backup_system 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/18cd51f7497643d49709c1a7bd387f23 for snapshot=snapshot_backup_system 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/0c4df14ef8d447179ad82847c1682b6f for snapshot=snapshot_backup_system 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/7ae25ee1260b436790453308710d0623, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/48d5803e75234c9f8c50d40123730dbe, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/2e78fea284d9486c91abac25af6428e0, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/a18fbc9163324723bd9ebd8c9c7c2829] hfiles 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f for snapshot=snapshot_backup_system 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60 for snapshot=snapshot_backup_system 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42 for snapshot=snapshot_backup_system 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/7ae25ee1260b436790453308710d0623 for snapshot=snapshot_backup_system 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/48d5803e75234c9f8c50d40123730dbe for snapshot=snapshot_backup_system 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/2e78fea284d9486c91abac25af6428e0 for snapshot=snapshot_backup_system 2023-11-12T09:32:55,796 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/a18fbc9163324723bd9ebd8c9c7c2829 for snapshot=snapshot_backup_system 2023-11-12T09:32:56,202 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:32:56,202 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-11-12T09:32:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4026): Remote procedure done, pid=51 2023-11-12T09:32:56,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:56,204 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:32:56,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-11-12T09:32:56,206 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:32:56,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb in 1.4020 sec 2023-11-12T09:32:56,206 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:32:56,207 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:32:56,207 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:32:56,208 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:32:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:32:56,618 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:32:56,629 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:32:56,629 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:32:56,630 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:32:56,630 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-11-12T09:32:56,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2430 sec 2023-11-12T09:32:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:32:58,497 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 50 completed 2023-11-12T09:32:58,497 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(115): Backup backup_1699781574344 started at 1699781578497. 2023-11-12T09:32:58,499 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(119): Backup session backup_1699781574344 has been started. 2023-11-12T09:32:58,504 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(268): For incremental backup, current table set is [test-1699781502011, ns2:test-16997815020111, ns3:test-16997815020112] 2023-11-12T09:32:58,507 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(76): StartCode 1699781546114for backupID backup_1699781574344 2023-11-12T09:32:58,507 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-11-12T09:32:58,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:32:58,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:32:58,508 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:32:58,508 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:32:58,509 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:32:58,509 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:32:58,509 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:58,509 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:58,512 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:32:58,512 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,512 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:32:58,512 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:32:58,512 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,512 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:32:58,513 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:58,513 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:58,513 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:32:58,513 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:58,513 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:32:58,514 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:32:58,514 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:32:58,514 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:32:58,514 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:32:58,515 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:32:58,515 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:32:58,515 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:32:58,517 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,517 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:58,518 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,518 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:58,518 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:58,518 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:32:58,518 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:32:58,518 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:58,518 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:58,519 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:58,519 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,520 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:58,520 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37711,1699781492776' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:32:58,520 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4d3fc4f4[Count = 0] remaining members to acquire global barrier 2023-11-12T09:32:58,520 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:32:58,520 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:58,522 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:58,522 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:58,522 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:58,522 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:32:58,522 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,522 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:32:58,522 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:32:58,522 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,522 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699781568998 highest: 1699781568998 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,522 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776:(num 1699781568998) roll requested 2023-11-12T09:32:58,534 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781578522, exclude list is [], retry=0 2023-11-12T09:32:58,538 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:32:58,540 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781568998 with entries=10, filesize=2.37 KB; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781578522 2023-11-12T09:32:58,540 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:32:58,540 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781568998 is not closed yet, will try archiving it next time 2023-11-12T09:32:58,540 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta:.meta(num 1699781569017) roll requested 2023-11-12T09:32:58,540 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781557624 to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781557624 2023-11-12T09:32:58,556 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781578540.meta, exclude list is [], retry=0 2023-11-12T09:32:58,559 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK] 2023-11-12T09:32:58,561 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781569017.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781578540.meta 2023-11-12T09:32:58,561 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43915,DS-d9986012-1171-4bc5-83f5-e236c8649a49,DISK]] 2023-11-12T09:32:58,561 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781569017.meta is not closed yet, will try archiving it next time 2023-11-12T09:32:58,562 DEBUG [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:32:58,563 INFO [rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699781578522 on jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,563 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781569017.meta to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781569017.meta 2023-11-12T09:32:58,572 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:32:58,572 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37711,1699781492776' in zk 2023-11-12T09:32:58,575 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:32:58,575 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,575 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:32:58,575 DEBUG [member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:32:58,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:58,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:58,577 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:58,577 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:58,577 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:58,578 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,578 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:58,578 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:58,579 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,579 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37711,1699781492776': 2023-11-12T09:32:58,579 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37711,1699781492776' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:32:58,579 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:32:58,579 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:32:58,579 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:32:58,579 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:32:58,579 INFO [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:32:58,582 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:58,582 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:32:58,582 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:58,582 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:58,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:58,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:58,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:32:58,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:32:58,583 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:32:58,583 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:32:58,583 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:32:58,583 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,583 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:58,584 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:32:58,584 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:32:58,584 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:58,585 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,585 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,585 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:32:58,585 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:32:58,586 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,593 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,593 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:32:58,593 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:58,593 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:32:58,593 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:32:58,593 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:32:58,593 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:32:58,593 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:32:58,593 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:58,593 DEBUG [(jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:32:58,593 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:32:58,593 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:32:58,593 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:32:58,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:32:58,594 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:32:58,594 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-12T09:32:58,594 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:32:58,594 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:32:58,695 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:32:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:32:58,698 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:37711=1699781546114} newestTimestamps: {jenkins-hbase3.apache.org:37711=1699781568998} 2023-11-12T09:32:58,699 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781495617.meta 2023-11-12T09:32:58,699 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781578540.meta 2023-11-12T09:32:58,699 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781568998 2023-11-12T09:32:58,699 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781578522 2023-11-12T09:32:58,700 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781512457.meta 2023-11-12T09:32:58,700 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781527575.meta 2023-11-12T09:32:58,700 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781536531.meta 2023-11-12T09:32:58,700 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781546132.meta 2023-11-12T09:32:58,700 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781557643.meta 2023-11-12T09:32:58,700 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta.1699781569017.meta 2023-11-12T09:32:59,110 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(147): Attempting to copy table info for:ns2:test-16997815020111 target: hdfs://localhost:38513/backupUT/backup_1699781574344/ns2/test-16997815020111 descriptor: 'ns2:test-16997815020111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:32:59,110 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:32:59,112 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(152): Starting to write region info for table ns2:test-16997815020111 2023-11-12T09:32:59,517 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(159): Finished writing region info for table ns2:test-16997815020111 2023-11-12T09:32:59,928 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(147): Attempting to copy table info for:ns3:test-16997815020112 target: hdfs://localhost:38513/backupUT/backup_1699781574344/ns3/test-16997815020112 descriptor: 'ns3:test-16997815020112', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:32:59,928 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:32:59,930 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(152): Starting to write region info for table ns3:test-16997815020112 2023-11-12T09:32:59,935 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(159): Finished writing region info for table ns3:test-16997815020112 2023-11-12T09:33:00,344 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(147): Attempting to copy table info for:test-1699781502011 target: hdfs://localhost:38513/backupUT/backup_1699781574344/default/test-1699781502011 descriptor: 'test-1699781502011', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:33:00,344 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-12T09:33:00,346 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(152): Starting to write region info for table test-1699781502011 2023-11-12T09:33:00,751 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(159): Finished writing region info for table test-1699781502011 2023-11-12T09:33:00,757 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:38513/backupUT/.tmp/backup_1699781574344 from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781568998;hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781557624 2023-11-12T09:33:00,757 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6b196c75 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:33:00,763 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@765e4f11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:33:00,764 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:33:00,766 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:47010, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:33:00,768 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-12T09:33:00,768 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1699781502011 2023-11-12T09:33:00,771 DEBUG [hconnection-0x17047171-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:33:00,772 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:53580, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:33:00,774 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1699781502011: test-1699781502011; 2023-11-12T09:33:00,774 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16997815020111 2023-11-12T09:33:00,775 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16997815020111: ns2:test-16997815020111; 2023-11-12T09:33:00,775 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns3:test-16997815020112 2023-11-12T09:33:00,777 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns3:test-16997815020112: ns3:test-16997815020112; 2023-11-12T09:33:00,777 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(694): Configuring 3 reduce partitions to match current region count for all tables 2023-11-12T09:33:00,777 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/hbase-staging/partitions_641099be-82d8-467c-9c5b-5de4e7e38c4b 2023-11-12T09:33:01,185 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:01,185 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:01,186 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:01,186 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,172 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop-8480442019348458080.jar 2023-11-12T09:33:02,172 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,173 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,173 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,174 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,174 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,174 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,175 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,175 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,175 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,176 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:33:02,177 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:33:02,177 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:33:02,178 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:33:02,178 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:33:02,179 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:33:02,179 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:33:02,179 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:33:02,180 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:33:02,180 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:33:02,181 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:33:02,181 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:33:02,182 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,182 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,182 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,183 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,183 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,183 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:33:02,184 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:33:02,184 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: default:test-1699781502011,ns2:test-16997815020111,ns3:test-16997815020112 2023-11-12T09:33:02,184 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:33:02,184 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6b196c75 to 127.0.0.1:55425 2023-11-12T09:33:02,184 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:33:02,185 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:33:03,145 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop-7563095681051994267.jar 2023-11-12T09:33:03,150 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:33:03,168 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781568998 for WAL files 2023-11-12T09:33:03,169 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781568998 2023-11-12T09:33:03,169 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781557624 for WAL files 2023-11-12T09:33:03,170 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781557624 2023-11-12T09:33:03,240 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781583206/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:33:03,240 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/mapreduce.cluster.local.dir/1699781583206/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:33:03,290 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781557624 for hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781557624 (-9223372036854775808:9223372036854775807) length:2574 2023-11-12T09:33:03,293 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:33:03,293 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:33:03,315 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781568998 for hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/WALs/jenkins-hbase3.apache.org,37711,1699781492776/jenkins-hbase3.apache.org%2C37711%2C1699781492776.1699781568998 (-9223372036854775808:9223372036854775807) length:2432 2023-11-12T09:33:03,318 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-12T09:33:03,318 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-12T09:33:04,273 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:38513/backupUT 2023-11-12T09:33:04,273 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1699781574344 2023-11-12T09:33:04,273 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-11-12T09:33:04,278 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:38513/backupUT/.tmp/backup_1699781574344, hdfs://localhost:38513/backupUT] 2023-11-12T09:33:04,300 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:33:04,876 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-11-12T09:33:04,878 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1699781574344 2023-11-12T09:33:04,878 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 0 bytes copied." 2023-11-12T09:33:04,878 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local2018447140_0013 completed: true true 2023-11-12T09:33:04,881 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=6753386 FILE: Number of bytes written=16012544 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=326544 HDFS: Number of bytes written=2475653 HDFS: Number of read operations=1076 HDFS: Number of large read operations=0 HDFS: Number of write operations=380 Map-Reduce Framework Map input records=2 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1838153728 File Input Format Counters Bytes Read=486 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=0 Bytes Expected=0 Files Copied=1 DIR_COPY=1 2023-11-12T09:33:04,881 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:38513/backupUT/.tmp/backup_1699781574344 to hdfs://localhost:38513/backupUT finished. 2023-11-12T09:33:04,894 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-11-12T09:33:04,895 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781574344 2023-11-12T09:33:04,899 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781553447 to construct the dependency. 2023-11-12T09:33:04,899 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781553447 2023-11-12T09:33:04,903 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781553447/.backup.manifest 2023-11-12T09:33:04,903 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781553447BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,903 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781543940 to construct the dependency. 2023-11-12T09:33:04,903 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781543940 2023-11-12T09:33:04,905 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:33:04,905 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781543940BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,905 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:33:04,905 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:33:04,907 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:33:04,907 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,907 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:33:04,907 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:33:04,909 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:33:04,909 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,909 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-11-12T09:33:04,909 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781574344 2023-11-12T09:33:04,911 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781553447 to construct the dependency. 2023-11-12T09:33:04,911 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781553447 2023-11-12T09:33:04,914 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781553447/.backup.manifest 2023-11-12T09:33:04,914 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781553447BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,914 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781543940 to construct the dependency. 2023-11-12T09:33:04,914 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781543940 2023-11-12T09:33:04,916 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:33:04,916 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781543940BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,916 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:33:04,917 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:33:04,918 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:33:04,918 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,918 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:33:04,919 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:33:04,920 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:33:04,920 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,920 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-11-12T09:33:04,926 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781574344/.backup.manifest 2023-11-12T09:33:04,926 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781574344 2023-11-12T09:33:04,928 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781553447 to construct the dependency. 2023-11-12T09:33:04,928 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781553447 2023-11-12T09:33:04,930 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781553447/.backup.manifest 2023-11-12T09:33:04,930 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781553447BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,930 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781543940 to construct the dependency. 2023-11-12T09:33:04,930 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781543940 2023-11-12T09:33:04,932 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:33:04,932 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781543940BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,932 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:33:04,933 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:33:04,934 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:33:04,934 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,934 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:33:04,934 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:33:04,936 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:33:04,936 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,936 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-11-12T09:33:04,936 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781574344 2023-11-12T09:33:04,938 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781553447 to construct the dependency. 2023-11-12T09:33:04,938 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781553447 2023-11-12T09:33:04,941 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781553447/.backup.manifest 2023-11-12T09:33:04,941 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781553447BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,941 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781543940 to construct the dependency. 2023-11-12T09:33:04,941 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781543940 2023-11-12T09:33:04,943 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:33:04,943 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781543940BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,943 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:33:04,943 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:33:04,945 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:33:04,945 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,945 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:33:04,945 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:33:04,947 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:33:04,947 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:04,947 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-11-12T09:33:05,355 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781574344/.backup.manifest 2023-11-12T09:33:05,355 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781574344 2023-11-12T09:33:05,358 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781553447 to construct the dependency. 2023-11-12T09:33:05,358 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781553447 2023-11-12T09:33:05,361 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781553447/.backup.manifest 2023-11-12T09:33:05,361 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781553447BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:05,361 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781543940 to construct the dependency. 2023-11-12T09:33:05,361 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781543940 2023-11-12T09:33:05,363 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:33:05,363 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781543940BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:05,363 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:33:05,363 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:33:05,365 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:33:05,365 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:05,365 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:33:05,366 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:33:05,368 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:33:05,368 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:05,368 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-11-12T09:33:05,368 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781574344 2023-11-12T09:33:05,370 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781553447 to construct the dependency. 2023-11-12T09:33:05,370 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781553447 2023-11-12T09:33:05,372 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781553447/.backup.manifest 2023-11-12T09:33:05,372 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781553447BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:05,372 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781543940 to construct the dependency. 2023-11-12T09:33:05,372 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781543940 2023-11-12T09:33:05,374 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:33:05,374 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781543940BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:05,374 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:33:05,374 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:33:05,375 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:33:05,375 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:05,375 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:33:05,376 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:33:05,377 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:33:05,377 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:05,377 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-11-12T09:33:05,383 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781574344/.backup.manifest 2023-11-12T09:33:05,383 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699781574344 2023-11-12T09:33:05,385 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781553447 to construct the dependency. 2023-11-12T09:33:05,385 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781553447 2023-11-12T09:33:05,387 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781553447/.backup.manifest 2023-11-12T09:33:05,387 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781553447BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:05,387 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781543940 to construct the dependency. 2023-11-12T09:33:05,387 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781543940 2023-11-12T09:33:05,389 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781543940/.backup.manifest 2023-11-12T09:33:05,389 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781543940BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:05,389 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781534329 to construct the dependency. 2023-11-12T09:33:05,389 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781534329 2023-11-12T09:33:05,391 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781534329/.backup.manifest 2023-11-12T09:33:05,391 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781534329BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:05,391 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(327): Current backup has an incremental backup ancestor, touching its image manifest in hdfs://localhost:38513/backupUT/backup_1699781523368 to construct the dependency. 2023-11-12T09:33:05,391 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:38513/backupUT/backup_1699781523368 2023-11-12T09:33:05,393 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1699781523368/.backup.manifest 2023-11-12T09:33:05,393 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(335): Last dependent incremental backup image: {BackupID=backup_1699781523368BackupDir=hdfs://localhost:38513/backupUT} 2023-11-12T09:33:05,393 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManager(340): Got 6 ancestors for the current backup. 2023-11-12T09:33:05,799 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:38513/backupUT/backup_1699781574344/.backup.manifest 2023-11-12T09:33:05,799 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(358): Backup backup_1699781574344 finished: type=INCREMENTAL,tablelist=ns2:test-16997815020111;ns3:test-16997815020112;test-1699781502011,targetRootDir=hdfs://localhost:38513/backupUT,startts=1699781578497,completets=1699781584894,bytescopied=0 2023-11-12T09:33:05,799 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:33:05,804 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:33:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:33:05,807 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:33:05,808 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:33:05,811 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.TableBackupClient(376): Backup backup_1699781574344 completed. 2023-11-12T09:33:05,822 ERROR [Listener at jenkins-hbase3.apache.org/35101 {}] backup.TestBackupMultipleDeletes(130): Delete backupIdInc2 2023-11-12T09:33:05,830 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:33:05,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:33:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699781585838 (current time:1699781585838). 2023-11-12T09:33:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:33:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:33:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x31e7b022 to 127.0.0.1:55425 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:33:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1043061, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:33:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:33:05,847 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36038, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:33:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x31e7b022 to 127.0.0.1:55425 2023-11-12T09:33:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:33:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:33:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1032): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:33:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-11-12T09:33:05,852 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:33:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-12T09:33:05,853 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:33:05,856 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot, current working dir path: hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:33:05,858 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:33:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-12T09:33:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-12T09:33:06,263 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:33:06,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb}] 2023-11-12T09:33:06,265 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:33:06,416 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:33:06,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37711 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-11-12T09:33:06,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:33:06,417 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing 0e93acef4885d53d7a3e4dd8fbdd16bb 2/2 column families, dataSize=1.75 KB heapSize=3.05 KB 2023-11-12T09:33:06,421 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/41a4350fe60644cfbd9f004f5bd8ef4a is 171, key is trslm:hdfs://localhost:38513/backupUT\x00ns2:test-16997815020111/meta:log-roll-map/1699781584885/Put/seqid=0 2023-11-12T09:33:06,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-12T09:33:06,827 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=758 B at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/41a4350fe60644cfbd9f004f5bd8ef4a 2023-11-12T09:33:06,836 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/d8d51868d17749e9915190e8795d50a8 is 461, key is session:backup_1699781574344/session:context/1699781585807/Put/seqid=0 2023-11-12T09:33:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-12T09:33:07,241 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.01 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/d8d51868d17749e9915190e8795d50a8 2023-11-12T09:33:07,247 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/41a4350fe60644cfbd9f004f5bd8ef4a as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/41a4350fe60644cfbd9f004f5bd8ef4a 2023-11-12T09:33:07,252 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/41a4350fe60644cfbd9f004f5bd8ef4a, entries=6, sequenceid=84, filesize=5.8 K 2023-11-12T09:33:07,253 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/d8d51868d17749e9915190e8795d50a8 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d8d51868d17749e9915190e8795d50a8 2023-11-12T09:33:07,258 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d8d51868d17749e9915190e8795d50a8, entries=2, sequenceid=84, filesize=5.4 K 2023-11-12T09:33:07,259 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~1.75 KB/1792, heapSize ~3.02 KB/3096, currentSize=0 B/0 for 0e93acef4885d53d7a3e4dd8fbdd16bb in 841ms, sequenceid=84, compaction requested=true 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for 0e93acef4885d53d7a3e4dd8fbdd16bb: 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. for snapshot_backup_system completed. 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/e99f19007bb0490ea42ad6f54a6a2f02, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/9c02b6ae5d514a4b9d2b2ffef3fc5209, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/18cd51f7497643d49709c1a7bd387f23, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/0c4df14ef8d447179ad82847c1682b6f, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/41a4350fe60644cfbd9f004f5bd8ef4a] hfiles 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/5ed1e945332b4ef09767e2ffece9f631 for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/28ae5f7615ab48e09813a26c19b80a8f for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/e99f19007bb0490ea42ad6f54a6a2f02 for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/9c02b6ae5d514a4b9d2b2ffef3fc5209 for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/18cd51f7497643d49709c1a7bd387f23 for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/0c4df14ef8d447179ad82847c1682b6f for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/41a4350fe60644cfbd9f004f5bd8ef4a for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/7ae25ee1260b436790453308710d0623, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/48d5803e75234c9f8c50d40123730dbe, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/2e78fea284d9486c91abac25af6428e0, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/a18fbc9163324723bd9ebd8c9c7c2829, hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d8d51868d17749e9915190e8795d50a8] hfiles 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d07cc77bb3f3424098b0a6f3fd359a0f for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/8195231b05ab4f6d978e10371472ca60 for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/50a204075f864a39805ebe7f35ecdb42 for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/7ae25ee1260b436790453308710d0623 for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/48d5803e75234c9f8c50d40123730dbe for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/2e78fea284d9486c91abac25af6428e0 for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/a18fbc9163324723bd9ebd8c9c7c2829 for snapshot=snapshot_backup_system 2023-11-12T09:33:07,259 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/d8d51868d17749e9915190e8795d50a8 for snapshot=snapshot_backup_system 2023-11-12T09:33:07,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:33:07,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-11-12T09:33:07,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4026): Remote procedure done, pid=53 2023-11-12T09:33:07,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:33:07,666 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:33:07,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-11-12T09:33:07,669 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:33:07,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure 0e93acef4885d53d7a3e4dd8fbdd16bb in 1.4040 sec 2023-11-12T09:33:07,669 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:33:07,670 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:33:07,670 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:33:07,671 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:33:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-12T09:33:08,083 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:33:08,095 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:33:08,095 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:33:08,096 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:33:08,096 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-11-12T09:33:08,097 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2460 sec 2023-11-12T09:33:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-12T09:33:09,959 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 52 completed 2023-11-12T09:33:09,964 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(227): Deleting backup backup_1699781534329 ... 2023-11-12T09:33:09,965 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(437): Delete log files: .backup.manifest 2023-11-12T09:33:09,966 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(437): Delete log files: _SUCCESS 2023-11-12T09:33:09,966 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(437): Delete log files: default 2023-11-12T09:33:09,966 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(437): Delete log files: ns2 2023-11-12T09:33:09,967 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(445): Trying to cleanup up target dir : backup_1699781534329 2023-11-12T09:33:09,967 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(460): No data has been found in hdfs://localhost:38513/backupUT/backup_1699781534329/ns2/test-16997815020111. 2023-11-12T09:33:09,968 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(467): hdfs://localhost:38513/backupUT/backup_1699781534329/ns2 is empty, remove it. 2023-11-12T09:33:09,968 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(460): No data has been found in hdfs://localhost:38513/backupUT/backup_1699781534329/default/test-1699781502011. 2023-11-12T09:33:09,969 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(467): hdfs://localhost:38513/backupUT/backup_1699781534329/default is empty, remove it. 2023-11-12T09:33:09,972 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(309): GetAffectedBackupInfos for: backup_1699781534329 table=ns2:test-16997815020111 2023-11-12T09:33:09,974 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1699781534329 table=ns2:test-16997815020111 added backup_1699781574344 tables={ns2:test-16997815020111,ns3:test-16997815020112,test-1699781502011} 2023-11-12T09:33:09,974 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1699781534329 table=ns2:test-16997815020111 added backup_1699781553447 tables={ns2:test-16997815020111,test-1699781502011} 2023-11-12T09:33:09,974 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1699781534329 table=ns2:test-16997815020111 added backup_1699781543940 tables={ns2:test-16997815020111,test-1699781502011} 2023-11-12T09:33:09,974 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(287): Remove ns2:test-16997815020111 from backup_1699781574344 tables={ns2:test-16997815020111,ns3:test-16997815020112,test-1699781502011} 2023-11-12T09:33:09,976 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:38513/backupUT/backup_1699781574344/ns2/test-16997815020111 done. 2023-11-12T09:33:09,976 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(287): Remove ns2:test-16997815020111 from backup_1699781553447 tables={ns2:test-16997815020111,test-1699781502011} 2023-11-12T09:33:09,978 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:38513/backupUT/backup_1699781553447/ns2/test-16997815020111 done. 2023-11-12T09:33:09,978 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(287): Remove ns2:test-16997815020111 from backup_1699781543940 tables={ns2:test-16997815020111,test-1699781502011} 2023-11-12T09:33:09,980 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:38513/backupUT/backup_1699781543940/ns2/test-16997815020111 done. 2023-11-12T09:33:09,982 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(309): GetAffectedBackupInfos for: backup_1699781534329 table=test-1699781502011 2023-11-12T09:33:09,984 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1699781534329 table=test-1699781502011 added backup_1699781574344 tables={ns3:test-16997815020112,test-1699781502011} 2023-11-12T09:33:09,984 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1699781534329 table=test-1699781502011 added backup_1699781553447 tables={test-1699781502011} 2023-11-12T09:33:09,984 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(326): GetAffectedBackupInfos for: backup_1699781534329 table=test-1699781502011 added backup_1699781543940 tables={test-1699781502011} 2023-11-12T09:33:09,984 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(287): Remove test-1699781502011 from backup_1699781574344 tables={ns3:test-16997815020112,test-1699781502011} 2023-11-12T09:33:09,986 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(354): Cleaning up backup data at hdfs://localhost:38513/backupUT/backup_1699781574344/default/test-1699781502011 done. 2023-11-12T09:33:09,986 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(287): Remove test-1699781502011 from backup_1699781553447 tables={test-1699781502011} 2023-11-12T09:33:09,986 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(293): Delete backup info backup_1699781553447 2023-11-12T09:33:09,989 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(437): Delete log files: .backup.manifest 2023-11-12T09:33:09,989 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(437): Delete log files: _SUCCESS 2023-11-12T09:33:09,990 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(437): Delete log files: default 2023-11-12T09:33:09,990 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(437): Delete log files: ns2 2023-11-12T09:33:09,991 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(445): Trying to cleanup up target dir : backup_1699781553447 2023-11-12T09:33:09,991 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(460): No data has been found in hdfs://localhost:38513/backupUT/backup_1699781553447/default/test-1699781502011. 2023-11-12T09:33:09,992 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(467): hdfs://localhost:38513/backupUT/backup_1699781553447/default is empty, remove it. 2023-11-12T09:33:09,992 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(287): Remove test-1699781502011 from backup_1699781543940 tables={test-1699781502011} 2023-11-12T09:33:09,992 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(293): Delete backup info backup_1699781543940 2023-11-12T09:33:09,994 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(437): Delete log files: .backup.manifest 2023-11-12T09:33:09,994 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(437): Delete log files: _SUCCESS 2023-11-12T09:33:09,995 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(437): Delete log files: default 2023-11-12T09:33:09,995 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(437): Delete log files: ns2 2023-11-12T09:33:09,996 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(445): Trying to cleanup up target dir : backup_1699781543940 2023-11-12T09:33:09,996 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(460): No data has been found in hdfs://localhost:38513/backupUT/backup_1699781543940/default/test-1699781502011. 2023-11-12T09:33:09,997 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.BackupUtils(467): hdfs://localhost:38513/backupUT/backup_1699781543940/default is empty, remove it. 2023-11-12T09:33:09,998 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-11-12T09:33:09,999 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-11-12T09:33:09,999 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupAdminImpl(276): Delete backup backup_1699781534329 completed. 2023-11-12T09:33:10,004 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1034): test-1699781502011 2023-11-12T09:33:10,004 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1034): ns2:test-16997815020111 2023-11-12T09:33:10,004 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1034): ns3:test-16997815020112 2023-11-12T09:33:10,006 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:33:10,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:33:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:33:10,012 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:33:10,012 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:33:10,014 ERROR [Listener at jenkins-hbase3.apache.org/35101 {}] backup.TestBackupMultipleDeletes(132): Delete backupIdInc2 done 2023-11-12T09:33:10,023 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:33:10,024 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4539b7f4 to 127.0.0.1:55425 2023-11-12T09:33:10,024 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:33:10,048 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.ResourceChecker(175): after: backup.TestBackupMultipleDeletes#testBackupMultipleDeletes Thread=591 (was 584) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2108273953_16 at /127.0.0.1:46416 [Waiting for operation #45] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-86427752_16 at /127.0.0.1:46348 [Receiving block BP-1130439829-172.31.12.81-1699781485924:blk_1073741960_1136] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1130439829-172.31.12.81-1699781485924:blk_1073741960_1136, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/cluster_7b0087cc-f951-95b8-513e-bf398edab43a/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1130439829-172.31.12.81-1699781485924:blk_1073741959_1135, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/cluster_7b0087cc-f951-95b8-513e-bf398edab43a/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2123121450_16 at /127.0.0.1:35180 [Waiting for operation #9] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,35261,1699781489344)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-86427752_16 at /127.0.0.1:46346 [Receiving block BP-1130439829-172.31.12.81-1699781485924:blk_1073741959_1135] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x39505ecb-shared-pool-31 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37711,1699781492776)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,37711,1699781492776' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2108273953_16 at /127.0.0.1:35160 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=706 (was 689) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=114 (was 254), ProcessCount=168 (was 168), AvailableMemoryMB=5132 (was 6194) 2023-11-12T09:33:10,048 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.ResourceChecker(130): Thread=591 is superior to 500 2023-11-12T09:33:10,055 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-11-12T09:33:10,055 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:33:10,055 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x577a5d63 to 127.0.0.1:55425 2023-11-12T09:33:10,055 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:33:10,056 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-11-12T09:33:10,056 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] util.JVMClusterUtil(257): Found active master hash=1679885594, stopped=false 2023-11-12T09:33:10,056 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] master.ServerManager(896): Cluster shutdown requested of master=jenkins-hbase3.apache.org,35261,1699781489344 2023-11-12T09:33:10,059 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-11-12T09:33:10,059 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:33:10,059 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2023-11-12T09:33:10,059 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:33:10,059 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] procedure2.ProcedureExecutor(630): Stopping 2023-11-12T09:33:10,059 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:33:10,059 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:33:10,060 DEBUG [Listener at jenkins-hbase3.apache.org/35101 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:33:10,060 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,37711,1699781492776' ***** 2023-11-12T09:33:10,060 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-11-12T09:33:10,060 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-12T09:33:10,073 INFO [RS:0;jenkins-hbase3:37711 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@267f8c07{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-12T09:33:10,078 INFO [RS:0;jenkins-hbase3:37711 {}] server.AbstractConnector(383): Stopped ServerConnector@24fa6f21{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:33:10,078 INFO [RS:0;jenkins-hbase3:37711 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-12T09:33:10,079 INFO [RS:0;jenkins-hbase3:37711 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@26457e15{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-12T09:33:10,080 INFO [RS:0;jenkins-hbase3:37711 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5ffd5097{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop.log.dir/,STOPPED} 2023-11-12T09:33:10,082 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HeapMemoryManager(220): Stopping 2023-11-12T09:33:10,082 INFO [RS:0;jenkins-hbase3:37711 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-11-12T09:33:10,082 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-11-12T09:33:10,082 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-11-12T09:33:10,082 INFO [RS:0;jenkins-hbase3:37711 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-11-12T09:33:10,083 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(3528): Received CLOSE for 13632229548fde4afa1b285f20ec0adf 2023-11-12T09:33:10,083 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(3528): Received CLOSE for 5d178aff93cac37e2dcbca54eac41805 2023-11-12T09:33:10,083 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(3528): Received CLOSE for f37756824c9283a5488eef653b620975 2023-11-12T09:33:10,083 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(3528): Received CLOSE for d2b50154a764d8ff14c1aceb1c414835 2023-11-12T09:33:10,083 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(3528): Received CLOSE for 0e93acef4885d53d7a3e4dd8fbdd16bb 2023-11-12T09:33:10,083 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(3528): Received CLOSE for 2e735f09d7b85fcf9fb65b6e7191fab4 2023-11-12T09:33:10,083 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(3528): Received CLOSE for 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:33:10,083 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:33:10,083 INFO [RS:0;jenkins-hbase3:37711 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:33:10,083 DEBUG [RS:0;jenkins-hbase3:37711 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:33:10,083 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-11-12T09:33:10,083 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-11-12T09:33:10,084 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-11-12T09:33:10,084 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-11-12T09:33:10,084 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 13632229548fde4afa1b285f20ec0adf, disabling compactions & flushes 2023-11-12T09:33:10,084 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:33:10,084 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:33:10,084 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. after waiting 0 ms 2023-11-12T09:33:10,084 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:33:10,084 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-11-12T09:33:10,084 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1583): Online Regions={13632229548fde4afa1b285f20ec0adf=ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf., 5d178aff93cac37e2dcbca54eac41805=backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805., f37756824c9283a5488eef653b620975=ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975., 1588230740=hbase:meta,,1.1588230740, d2b50154a764d8ff14c1aceb1c414835=test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835., 0e93acef4885d53d7a3e4dd8fbdd16bb=backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb., 2e735f09d7b85fcf9fb65b6e7191fab4=hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4., 59d601d3d6dbb1192a7f8ee12e4c9815=ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815.} 2023-11-12T09:33:10,085 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-12T09:33:10,085 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-12T09:33:10,085 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-12T09:33:10,085 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-12T09:33:10,085 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-12T09:33:10,088 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=9.25 KB heapSize=16.20 KB 2023-11-12T09:33:10,088 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1609): Waiting on 0e93acef4885d53d7a3e4dd8fbdd16bb, 13632229548fde4afa1b285f20ec0adf, 1588230740, 2e735f09d7b85fcf9fb65b6e7191fab4, 59d601d3d6dbb1192a7f8ee12e4c9815, 5d178aff93cac37e2dcbca54eac41805, d2b50154a764d8ff14c1aceb1c414835, f37756824c9283a5488eef653b620975 2023-11-12T09:33:10,092 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns3/test-16997815020112/13632229548fde4afa1b285f20ec0adf/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2023-11-12T09:33:10,093 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:33:10,093 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:33:10,094 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 13632229548fde4afa1b285f20ec0adf: 2023-11-12T09:33:10,094 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16997815020112,,1699781505482.13632229548fde4afa1b285f20ec0adf. 2023-11-12T09:33:10,095 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 5d178aff93cac37e2dcbca54eac41805, disabling compactions & flushes 2023-11-12T09:33:10,095 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. 2023-11-12T09:33:10,095 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. 2023-11-12T09:33:10,095 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. after waiting 0 ms 2023-11-12T09:33:10,095 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. 2023-11-12T09:33:10,099 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system_bulk/5d178aff93cac37e2dcbca54eac41805/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:33:10,099 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:33:10,100 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. 2023-11-12T09:33:10,100 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 5d178aff93cac37e2dcbca54eac41805: 2023-11-12T09:33:10,100 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1699781509068.5d178aff93cac37e2dcbca54eac41805. 2023-11-12T09:33:10,101 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing f37756824c9283a5488eef653b620975, disabling compactions & flushes 2023-11-12T09:33:10,101 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. 2023-11-12T09:33:10,102 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. 2023-11-12T09:33:10,102 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. after waiting 0 ms 2023-11-12T09:33:10,102 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. 2023-11-12T09:33:10,106 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns4/test-16997815020113/f37756824c9283a5488eef653b620975/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:33:10,106 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:33:10,108 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-11-12T09:33:10,108 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. 2023-11-12T09:33:10,108 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for f37756824c9283a5488eef653b620975: 2023-11-12T09:33:10,108 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16997815020113,,1699781506610.f37756824c9283a5488eef653b620975. 2023-11-12T09:33:10,108 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing d2b50154a764d8ff14c1aceb1c414835, disabling compactions & flushes 2023-11-12T09:33:10,108 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:33:10,108 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:33:10,108 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. after waiting 0 ms 2023-11-12T09:33:10,108 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:33:10,109 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing d2b50154a764d8ff14c1aceb1c414835 1/1 column families, dataSize=7.13 KB heapSize=21.91 KB 2023-11-12T09:33:10,118 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/.tmp/f/3caec5aea39f49e7b677f2a9da3d758a is 42, key is row-t1100/f:q1/1699781543858/Put/seqid=0 2023-11-12T09:33:10,122 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740/.tmp/info/ca37e8afaad04a7bbe8a3565a64e85aa is 159, key is ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815./info:regioninfo/1699781505172/Put/seqid=0 2023-11-12T09:33:10,123 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/.tmp/f/3caec5aea39f49e7b677f2a9da3d758a 2023-11-12T09:33:10,133 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/.tmp/f/3caec5aea39f49e7b677f2a9da3d758a as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/f/3caec5aea39f49e7b677f2a9da3d758a 2023-11-12T09:33:10,138 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/f/3caec5aea39f49e7b677f2a9da3d758a, entries=198, sequenceid=304, filesize=12.6 K 2023-11-12T09:33:10,139 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~7.13 KB/7304, heapSize ~21.89 KB/22416, currentSize=0 B/0 for d2b50154a764d8ff14c1aceb1c414835 in 31ms, sequenceid=304, compaction requested=false 2023-11-12T09:33:10,148 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/default/test-1699781502011/d2b50154a764d8ff14c1aceb1c414835/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=1 2023-11-12T09:33:10,148 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:33:10,149 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:33:10,149 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for d2b50154a764d8ff14c1aceb1c414835: 2023-11-12T09:33:10,149 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1699781502011,,1699781502495.d2b50154a764d8ff14c1aceb1c414835. 2023-11-12T09:33:10,149 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 0e93acef4885d53d7a3e4dd8fbdd16bb, disabling compactions & flushes 2023-11-12T09:33:10,149 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:33:10,149 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:33:10,150 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. after waiting 0 ms 2023-11-12T09:33:10,150 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:33:10,150 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 0e93acef4885d53d7a3e4dd8fbdd16bb 2/2 column families, dataSize=1.94 KB heapSize=3.32 KB 2023-11-12T09:33:10,153 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/65dc055056da499b921e2e5ebc5eeb58 is 96, key is incrbackupset:hdfs://localhost:38513/backupUT/meta:ns2:test-16997815020111/1699781590005/Put/seqid=0 2023-11-12T09:33:10,289 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1609): Waiting on 0e93acef4885d53d7a3e4dd8fbdd16bb, 1588230740, 2e735f09d7b85fcf9fb65b6e7191fab4, 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:33:10,450 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-11-12T09:33:10,450 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-11-12T09:33:10,489 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1609): Waiting on 0e93acef4885d53d7a3e4dd8fbdd16bb, 1588230740, 2e735f09d7b85fcf9fb65b6e7191fab4, 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:33:10,531 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740/.tmp/info/ca37e8afaad04a7bbe8a3565a64e85aa 2023-11-12T09:33:10,550 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740/.tmp/table/80f31b7fadc84648b03c2e4a1b22439e is 59, key is ns2:test-16997815020111/table:state/1699781505182/Put/seqid=0 2023-11-12T09:33:10,557 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=308 B at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/65dc055056da499b921e2e5ebc5eeb58 2023-11-12T09:33:10,563 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 65dc055056da499b921e2e5ebc5eeb58 2023-11-12T09:33:10,563 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16997815020113 2023-11-12T09:33:10,565 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-11-12T09:33:10,565 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16997815020112 2023-11-12T09:33:10,566 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/75acbbb98bfc406d8c0abb99b109936c is 398, key is session:backup_1699781574344/session:context/1699781589985/Put/seqid=0 2023-11-12T09:33:10,566 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1699781502011 2023-11-12T09:33:10,689 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1609): Waiting on 0e93acef4885d53d7a3e4dd8fbdd16bb, 1588230740, 2e735f09d7b85fcf9fb65b6e7191fab4, 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:33:10,889 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1609): Waiting on 0e93acef4885d53d7a3e4dd8fbdd16bb, 1588230740, 2e735f09d7b85fcf9fb65b6e7191fab4, 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:33:10,954 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=714 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740/.tmp/table/80f31b7fadc84648b03c2e4a1b22439e 2023-11-12T09:33:10,960 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740/.tmp/info/ca37e8afaad04a7bbe8a3565a64e85aa as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740/info/ca37e8afaad04a7bbe8a3565a64e85aa 2023-11-12T09:33:10,965 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740/info/ca37e8afaad04a7bbe8a3565a64e85aa, entries=70, sequenceid=39, filesize=14.1 K 2023-11-12T09:33:10,966 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740/.tmp/table/80f31b7fadc84648b03c2e4a1b22439e as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740/table/80f31b7fadc84648b03c2e4a1b22439e 2023-11-12T09:33:10,972 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740/table/80f31b7fadc84648b03c2e4a1b22439e, entries=14, sequenceid=39, filesize=5.8 K 2023-11-12T09:33:10,973 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~9.25 KB/9474, heapSize ~15.92 KB/16304, currentSize=0 B/0 for 1588230740 in 888ms, sequenceid=39, compaction requested=false 2023-11-12T09:33:10,977 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.64 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/75acbbb98bfc406d8c0abb99b109936c 2023-11-12T09:33:10,983 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/meta/1588230740/recovered.edits/42.seqid, newMaxSeqId=42, maxSeqId=1 2023-11-12T09:33:10,984 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:33:10,984 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-12T09:33:10,985 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-12T09:33:10,985 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-12T09:33:10,985 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-11-12T09:33:10,985 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 75acbbb98bfc406d8c0abb99b109936c 2023-11-12T09:33:10,986 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/meta/65dc055056da499b921e2e5ebc5eeb58 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/65dc055056da499b921e2e5ebc5eeb58 2023-11-12T09:33:10,990 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 65dc055056da499b921e2e5ebc5eeb58 2023-11-12T09:33:10,990 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/meta/65dc055056da499b921e2e5ebc5eeb58, entries=4, sequenceid=97, filesize=5.4 K 2023-11-12T09:33:10,991 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/.tmp/session/75acbbb98bfc406d8c0abb99b109936c as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/75acbbb98bfc406d8c0abb99b109936c 2023-11-12T09:33:10,996 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 75acbbb98bfc406d8c0abb99b109936c 2023-11-12T09:33:10,996 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/session/75acbbb98bfc406d8c0abb99b109936c, entries=5, sequenceid=97, filesize=5.7 K 2023-11-12T09:33:10,997 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~1.94 KB/1990, heapSize ~3.29 KB/3368, currentSize=0 B/0 for 0e93acef4885d53d7a3e4dd8fbdd16bb in 847ms, sequenceid=97, compaction requested=true 2023-11-12T09:33:11,041 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/backup/system/0e93acef4885d53d7a3e4dd8fbdd16bb/recovered.edits/100.seqid, newMaxSeqId=100, maxSeqId=1 2023-11-12T09:33:11,042 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:33:11,043 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:33:11,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 0e93acef4885d53d7a3e4dd8fbdd16bb: 2023-11-12T09:33:11,043 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1699781507939.0e93acef4885d53d7a3e4dd8fbdd16bb. 2023-11-12T09:33:11,044 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 2e735f09d7b85fcf9fb65b6e7191fab4, disabling compactions & flushes 2023-11-12T09:33:11,044 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. 2023-11-12T09:33:11,044 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. 2023-11-12T09:33:11,044 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. after waiting 0 ms 2023-11-12T09:33:11,044 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. 2023-11-12T09:33:11,044 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 2e735f09d7b85fcf9fb65b6e7191fab4 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-11-12T09:33:11,057 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/namespace/2e735f09d7b85fcf9fb65b6e7191fab4/.tmp/info/74e7d5732e88409fb095c1e4441e5c9f is 45, key is default/info:d/1699781496875/Put/seqid=0 2023-11-12T09:33:11,090 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-11-12T09:33:11,090 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1583): Online Regions={2e735f09d7b85fcf9fb65b6e7191fab4=hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4., 59d601d3d6dbb1192a7f8ee12e4c9815=ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815.} 2023-11-12T09:33:11,090 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1609): Waiting on 2e735f09d7b85fcf9fb65b6e7191fab4, 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:33:11,290 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1609): Waiting on 2e735f09d7b85fcf9fb65b6e7191fab4, 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:33:11,462 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/namespace/2e735f09d7b85fcf9fb65b6e7191fab4/.tmp/info/74e7d5732e88409fb095c1e4441e5c9f 2023-11-12T09:33:11,469 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/namespace/2e735f09d7b85fcf9fb65b6e7191fab4/.tmp/info/74e7d5732e88409fb095c1e4441e5c9f as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/namespace/2e735f09d7b85fcf9fb65b6e7191fab4/info/74e7d5732e88409fb095c1e4441e5c9f 2023-11-12T09:33:11,475 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/namespace/2e735f09d7b85fcf9fb65b6e7191fab4/info/74e7d5732e88409fb095c1e4441e5c9f, entries=7, sequenceid=11, filesize=5.1 K 2023-11-12T09:33:11,476 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 2e735f09d7b85fcf9fb65b6e7191fab4 in 432ms, sequenceid=11, compaction requested=false 2023-11-12T09:33:11,482 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/hbase/namespace/2e735f09d7b85fcf9fb65b6e7191fab4/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-11-12T09:33:11,483 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:33:11,483 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. 2023-11-12T09:33:11,483 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 2e735f09d7b85fcf9fb65b6e7191fab4: 2023-11-12T09:33:11,483 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1699781495900.2e735f09d7b85fcf9fb65b6e7191fab4. 2023-11-12T09:33:11,484 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 59d601d3d6dbb1192a7f8ee12e4c9815, disabling compactions & flushes 2023-11-12T09:33:11,484 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:33:11,484 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:33:11,484 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. after waiting 0 ms 2023-11-12T09:33:11,484 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:33:11,484 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 59d601d3d6dbb1192a7f8ee12e4c9815 1/1 column families, dataSize=7.13 KB heapSize=21.91 KB 2023-11-12T09:33:11,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/.tmp/f/b9af0320972445fd8f6c3e039f55d628 is 42, key is row-t1100/f:q1/1699781553364/Put/seqid=0 2023-11-12T09:33:11,490 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1609): Waiting on 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:33:11,691 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1609): Waiting on 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:33:11,891 DEBUG [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1609): Waiting on 59d601d3d6dbb1192a7f8ee12e4c9815 2023-11-12T09:33:11,894 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.13 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/.tmp/f/b9af0320972445fd8f6c3e039f55d628 2023-11-12T09:33:11,901 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/.tmp/f/b9af0320972445fd8f6c3e039f55d628 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/f/b9af0320972445fd8f6c3e039f55d628 2023-11-12T09:33:11,905 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/f/b9af0320972445fd8f6c3e039f55d628, entries=198, sequenceid=304, filesize=12.6 K 2023-11-12T09:33:11,906 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~7.13 KB/7304, heapSize ~21.89 KB/22416, currentSize=0 B/0 for 59d601d3d6dbb1192a7f8ee12e4c9815 in 422ms, sequenceid=304, compaction requested=false 2023-11-12T09:33:11,912 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/data/ns2/test-16997815020111/59d601d3d6dbb1192a7f8ee12e4c9815/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=1 2023-11-12T09:33:11,912 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:33:11,913 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:33:11,913 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 59d601d3d6dbb1192a7f8ee12e4c9815: 2023-11-12T09:33:11,913 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16997815020111,,1699781504769.59d601d3d6dbb1192a7f8ee12e4c9815. 2023-11-12T09:33:12,092 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,37711,1699781492776; all regions closed. 2023-11-12T09:33:12,099 DEBUG [RS:0;jenkins-hbase3:37711 {}] wal.AbstractFSWAL(1062): Moved 2 WAL file(s) to /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs 2023-11-12T09:33:12,099 INFO [RS:0;jenkins-hbase3:37711 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776.meta:.meta(num 1699781578540) 2023-11-12T09:33:12,109 DEBUG [RS:0;jenkins-hbase3:37711 {}] wal.AbstractFSWAL(1062): Moved 7 WAL file(s) to /user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/oldWALs 2023-11-12T09:33:12,109 INFO [RS:0;jenkins-hbase3:37711 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C37711%2C1699781492776:(num 1699781578522) 2023-11-12T09:33:12,109 DEBUG [RS:0;jenkins-hbase3:37711 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:33:12,109 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.LeaseManager(133): Closed leases 2023-11-12T09:33:12,110 INFO [RS:0;jenkins-hbase3:37711 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-11-12T09:33:12,110 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-12T09:33:12,112 INFO [RS:0;jenkins-hbase3:37711 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:37711 2023-11-12T09:33:12,118 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/jenkins-hbase3.apache.org,37711,1699781492776 2023-11-12T09:33:12,118 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-11-12T09:33:12,121 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,37711,1699781492776] 2023-11-12T09:33:12,122 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,37711,1699781492776; numProcessing=1 2023-11-12T09:33:12,123 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /1/draining/jenkins-hbase3.apache.org,37711,1699781492776 already deleted, retry=false 2023-11-12T09:33:12,123 INFO [RegionServerTracker-0 {}] master.ServerManager(561): Cluster shutdown set; jenkins-hbase3.apache.org,37711,1699781492776 expired; onlineServers=0 2023-11-12T09:33:12,123 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,35261,1699781489344' ***** 2023-11-12T09:33:12,123 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-11-12T09:33:12,124 DEBUG [M:0;jenkins-hbase3:35261 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@631c3e96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-11-12T09:33:12,124 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-12T09:33:12,153 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2023-11-12T09:33:12,155 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:33:12,161 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-12T09:33:12,170 INFO [M:0;jenkins-hbase3:35261 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@6795993c{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-12T09:33:12,170 INFO [M:0;jenkins-hbase3:35261 {}] server.AbstractConnector(383): Stopped ServerConnector@99139ca{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:33:12,170 INFO [M:0;jenkins-hbase3:35261 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-12T09:33:12,171 INFO [M:0;jenkins-hbase3:35261 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@143113d4{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-12T09:33:12,172 INFO [M:0;jenkins-hbase3:35261 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@253fe48{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/hadoop.log.dir/,STOPPED} 2023-11-12T09:33:12,172 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,35261,1699781489344 2023-11-12T09:33:12,172 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,35261,1699781489344; all regions closed. 2023-11-12T09:33:12,172 DEBUG [M:0;jenkins-hbase3:35261 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:33:12,172 INFO [M:0;jenkins-hbase3:35261 {}] master.HMaster(1666): Stopping master jetty server 2023-11-12T09:33:12,173 INFO [M:0;jenkins-hbase3:35261 {}] server.AbstractConnector(383): Stopped ServerConnector@7843ecc9{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:33:12,173 DEBUG [M:0;jenkins-hbase3:35261 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-11-12T09:33:12,173 INFO [M:0;jenkins-hbase3:35261 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-11-12T09:33:12,173 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-11-12T09:33:12,173 DEBUG [M:0;jenkins-hbase3:35261 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x341b23bc to 127.0.0.1:55425 2023-11-12T09:33:12,173 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781494208 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699781494208,5,FailOnTimeoutGroup] 2023-11-12T09:33:12,173 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781494210 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699781494210,5,FailOnTimeoutGroup] 2023-11-12T09:33:12,174 DEBUG [M:0;jenkins-hbase3:35261 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:33:12,174 DEBUG [M:0;jenkins-hbase3:35261 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-11-12T09:33:12,174 INFO [M:0;jenkins-hbase3:35261 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-11-12T09:33:12,174 DEBUG [M:0;jenkins-hbase3:35261 {}] master.HMaster(1689): Stopping service threads 2023-11-12T09:33:12,174 INFO [M:0;jenkins-hbase3:35261 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-11-12T09:33:12,174 ERROR [M:0;jenkins-hbase3:35261 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-11-12T09:33:12,175 INFO [M:0;jenkins-hbase3:35261 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-11-12T09:33:12,175 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-11-12T09:33:12,175 DEBUG [M:0;jenkins-hbase3:35261 {}] zookeeper.ZKUtil(347): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2023-11-12T09:33:12,175 WARN [M:0;jenkins-hbase3:35261 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-11-12T09:33:12,175 INFO [M:0;jenkins-hbase3:35261 {}] assignment.AssignmentManager(383): Stopping assignment manager 2023-11-12T09:33:12,176 INFO [M:0;jenkins-hbase3:35261 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-11-12T09:33:12,176 DEBUG [M:0;jenkins-hbase3:35261 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-12T09:33:12,188 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:33:12,188 DEBUG [M:0;jenkins-hbase3:35261 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:33:12,188 DEBUG [M:0;jenkins-hbase3:35261 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-12T09:33:12,188 DEBUG [M:0;jenkins-hbase3:35261 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:33:12,188 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=202.04 KB heapSize=244.93 KB 2023-11-12T09:33:12,200 DEBUG [M:0;jenkins-hbase3:35261 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/867ba18793a84436a7c45b826814a6cf is 88, key is hbase:meta,,1/info:sn/1699781495224/Put/seqid=0 2023-11-12T09:33:12,204 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/867ba18793a84436a7c45b826814a6cf 2023-11-12T09:33:12,221 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-12T09:33:12,221 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37711-0x1004cf91cf70001, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-12T09:33:12,221 INFO [RS:0;jenkins-hbase3:37711 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,37711,1699781492776; zookeeper connection closed. 2023-11-12T09:33:12,221 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@65154612 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@65154612 2023-11-12T09:33:12,221 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-11-12T09:33:12,225 DEBUG [M:0;jenkins-hbase3:35261 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3581e4bd4e7f42a7a27ffd3a88bdd882 is 987, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1699781509858/Put/seqid=0 2023-11-12T09:33:12,630 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=201.45 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3581e4bd4e7f42a7a27ffd3a88bdd882 2023-11-12T09:33:12,635 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3581e4bd4e7f42a7a27ffd3a88bdd882 2023-11-12T09:33:12,648 DEBUG [M:0;jenkins-hbase3:35261 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/710d924bd8114394b2b2328f4d79f08c is 82, key is jenkins-hbase3.apache.org,37711,1699781492776/rs:state/1699781494308/Put/seqid=0 2023-11-12T09:33:13,053 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/710d924bd8114394b2b2328f4d79f08c 2023-11-12T09:33:13,058 DEBUG [M:0;jenkins-hbase3:35261 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/867ba18793a84436a7c45b826814a6cf as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/867ba18793a84436a7c45b826814a6cf 2023-11-12T09:33:13,063 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/867ba18793a84436a7c45b826814a6cf, entries=8, sequenceid=508, filesize=5.5 K 2023-11-12T09:33:13,064 DEBUG [M:0;jenkins-hbase3:35261 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3581e4bd4e7f42a7a27ffd3a88bdd882 as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3581e4bd4e7f42a7a27ffd3a88bdd882 2023-11-12T09:33:13,069 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3581e4bd4e7f42a7a27ffd3a88bdd882 2023-11-12T09:33:13,069 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3581e4bd4e7f42a7a27ffd3a88bdd882, entries=53, sequenceid=508, filesize=16.6 K 2023-11-12T09:33:13,070 DEBUG [M:0;jenkins-hbase3:35261 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/710d924bd8114394b2b2328f4d79f08c as hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/710d924bd8114394b2b2328f4d79f08c 2023-11-12T09:33:13,074 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:38513/user/jenkins/test-data/d4553cd2-0e44-fb89-8afe-09b6e8dd7d7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/710d924bd8114394b2b2328f4d79f08c, entries=1, sequenceid=508, filesize=5.1 K 2023-11-12T09:33:13,075 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.HRegion(3022): Finished flush of dataSize ~202.04 KB/206893, heapSize ~244.63 KB/250504, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 887ms, sequenceid=508, compaction requested=false 2023-11-12T09:33:13,080 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:33:13,080 DEBUG [M:0;jenkins-hbase3:35261 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:33:13,083 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-12T09:33:13,084 INFO [M:0;jenkins-hbase3:35261 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-11-12T09:33:13,084 INFO [M:0;jenkins-hbase3:35261 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-11-12T09:33:13,084 INFO [M:0;jenkins-hbase3:35261 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:35261 2023-11-12T09:33:13,087 DEBUG [M:0;jenkins-hbase3:35261 {}] zookeeper.RecoverableZooKeeper(224): Node /1/rs/jenkins-hbase3.apache.org,35261,1699781489344 already deleted, retry=false 2023-11-12T09:33:13,189 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-12T09:33:13,189 INFO [M:0;jenkins-hbase3:35261 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,35261,1699781489344; zookeeper connection closed. 2023-11-12T09:33:13,189 DEBUG [Listener at localhost/32839-EventThread {}] zookeeper.ZKWatcher(604): master:35261-0x1004cf91cf70000, quorum=127.0.0.1:55425, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2023-11-12T09:33:13,196 WARN [Listener at jenkins-hbase3.apache.org/35101 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-12T09:33:13,203 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:33:13,313 WARN [BP-1130439829-172.31.12.81-1699781485924 heartbeating to localhost/127.0.0.1:38513 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-12T09:33:13,313 WARN [BP-1130439829-172.31.12.81-1699781485924 heartbeating to localhost/127.0.0.1:38513 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1130439829-172.31.12.81-1699781485924 (Datanode Uuid 3750ebf4-1eec-4283-b25d-5998f5bf7e75) service to localhost/127.0.0.1:38513 2023-11-12T09:33:13,315 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/cluster_7b0087cc-f951-95b8-513e-bf398edab43a/dfs/data/data1/current/BP-1130439829-172.31.12.81-1699781485924 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:33:13,315 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/2db05a3c-bcc8-258b-0f6f-f6d1b70d688b/cluster_7b0087cc-f951-95b8-513e-bf398edab43a/dfs/data/data2/current/BP-1130439829-172.31.12.81-1699781485924 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:33:13,341 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:33:13,463 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2023-11-12T09:33:13,535 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-11-12T09:33:13,535 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-11-12T09:33:13,545 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:33:13,663 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:33:13,776 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:33:13,776 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:33:13,886 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-11-12T09:33:13,896 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-11-12T09:33:13,897 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:33:13,908 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:33:14,011 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:33:14,011 INFO [Listener at jenkins-hbase3.apache.org/35101 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped