2023-11-12 09:21:25,307 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@f5958c9 2023-11-12 09:21:25,320 main DEBUG Took 0.010472 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-11-12 09:21:25,320 main DEBUG PluginManager 'Core' found 129 plugins 2023-11-12 09:21:25,320 main DEBUG PluginManager 'Level' found 0 plugins 2023-11-12 09:21:25,321 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-11-12 09:21:25,323 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:21:25,331 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-11-12 09:21:25,343 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,345 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:21:25,345 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,345 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:21:25,346 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,346 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:21:25,347 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,347 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:21:25,347 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,347 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:21:25,348 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,348 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:21:25,349 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,349 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:21:25,349 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,349 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:21:25,350 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,350 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:21:25,350 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,351 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:21:25,351 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,351 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:21:25,352 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,352 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-12 09:21:25,352 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,352 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-11-12 09:21:25,354 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-12 09:21:25,356 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-11-12 09:21:25,357 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-11-12 09:21:25,358 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-11-12 09:21:25,359 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-11-12 09:21:25,359 main DEBUG PluginManager 'Converter' found 47 plugins 2023-11-12 09:21:25,374 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-11-12 09:21:25,377 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-11-12 09:21:25,378 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-11-12 09:21:25,378 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-11-12 09:21:25,378 main DEBUG createAppenders(={Console}) 2023-11-12 09:21:25,379 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@f5958c9 initialized 2023-11-12 09:21:25,379 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@f5958c9 2023-11-12 09:21:25,380 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@f5958c9 OK. 2023-11-12 09:21:25,380 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-11-12 09:21:25,381 main DEBUG OutputStream closed 2023-11-12 09:21:25,381 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-11-12 09:21:25,381 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-11-12 09:21:25,381 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@dbf57b3 OK 2023-11-12 09:21:25,430 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-11-12 09:21:25,432 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-11-12 09:21:25,433 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-11-12 09:21:25,435 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-11-12 09:21:25,435 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-11-12 09:21:25,435 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-11-12 09:21:25,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-11-12 09:21:25,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-11-12 09:21:25,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-11-12 09:21:25,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-11-12 09:21:25,437 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-11-12 09:21:25,437 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-11-12 09:21:25,438 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-11-12 09:21:25,438 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-11-12 09:21:25,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-11-12 09:21:25,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-11-12 09:21:25,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-11-12 09:21:25,440 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-11-12 09:21:25,443 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-12 09:21:25,444 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@63a12c68) with optional ClassLoader: null 2023-11-12 09:21:25,444 main DEBUG Shutdown hook enabled. Registering a new one. 2023-11-12 09:21:25,445 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@63a12c68] started OK. 2023-11-12T09:21:25,478 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures timeout: 13 mins 2023-11-12 09:21:25,482 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-11-12 09:21:25,482 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-12T09:21:25,785 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2 2023-11-12T09:21:25,790 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-12T09:21:25,792 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:21:25,794 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-11-12T09:21:25,797 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/cluster_c9d24771-f798-4b7f-5e55-10c183a2c0d9, deleteOnExit=true 2023-11-12T09:21:25,797 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-11-12T09:21:25,798 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/test.cache.data in system properties and HBase conf 2023-11-12T09:21:25,798 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/hadoop.tmp.dir in system properties and HBase conf 2023-11-12T09:21:25,798 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/hadoop.log.dir in system properties and HBase conf 2023-11-12T09:21:25,799 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-12T09:21:25,799 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-12T09:21:25,799 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-12T09:21:25,909 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-11-12T09:21:26,418 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-11-12T09:21:26,423 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:21:26,423 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:21:26,424 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-12T09:21:26,424 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:21:26,425 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-12T09:21:26,425 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-12T09:21:26,426 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:21:26,426 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:21:26,426 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-12T09:21:26,426 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/nfs.dump.dir in system properties and HBase conf 2023-11-12T09:21:26,427 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/java.io.tmpdir in system properties and HBase conf 2023-11-12T09:21:26,427 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:21:26,427 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-12T09:21:26,427 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-11-12T09:21:26,866 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:21:26,870 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:21:27,163 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-11-12T09:21:27,298 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@3bdf5845 via org.mortbay.log.Slf4jLog 2023-11-12T09:21:27,314 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:21:27,368 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:21:27,412 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/java.io.tmpdir/Jetty_localhost_36155_hdfs____8ukhrs/webapp 2023-11-12T09:21:27,540 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36155 2023-11-12T09:21:27,564 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-12T09:21:27,564 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-12T09:21:28,142 WARN [Listener at localhost/37959 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:21:28,205 WARN [Listener at localhost/37959 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-12T09:21:28,222 WARN [Listener at localhost/37959 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:21:28,228 INFO [Listener at localhost/37959 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:21:28,234 INFO [Listener at localhost/37959 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/java.io.tmpdir/Jetty_localhost_41141_datanode____.8o1n7s/webapp 2023-11-12T09:21:28,330 INFO [Listener at localhost/37959 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41141 2023-11-12T09:21:28,656 WARN [Listener at localhost/45829 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-12T09:21:28,951 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-12T09:21:29,103 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x31d69d52b7b34cb9: Processing first storage report for DS-2ab59756-3d8b-485f-95c2-dd011ce9885e from datanode 41ceac43-a147-4903-993b-0fa32b96298f 2023-11-12T09:21:29,105 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x31d69d52b7b34cb9: from storage DS-2ab59756-3d8b-485f-95c2-dd011ce9885e node DatanodeRegistration(127.0.0.1:41591, datanodeUuid=41ceac43-a147-4903-993b-0fa32b96298f, infoPort=43973, infoSecurePort=0, ipcPort=45829, storageInfo=lv=-57;cid=testClusterID;nsid=1759080153;c=1699780886949), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-11-12T09:21:29,105 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x31d69d52b7b34cb9: Processing first storage report for DS-2208a229-bc17-4f85-834e-802f1bd1403e from datanode 41ceac43-a147-4903-993b-0fa32b96298f 2023-11-12T09:21:29,105 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x31d69d52b7b34cb9: from storage DS-2208a229-bc17-4f85-834e-802f1bd1403e node DatanodeRegistration(127.0.0.1:41591, datanodeUuid=41ceac43-a147-4903-993b-0fa32b96298f, infoPort=43973, infoSecurePort=0, ipcPort=45829, storageInfo=lv=-57;cid=testClusterID;nsid=1759080153;c=1699780886949), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2023-11-12T09:21:29,150 DEBUG [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2 2023-11-12T09:21:29,294 INFO [Listener at localhost/45829 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/cluster_c9d24771-f798-4b7f-5e55-10c183a2c0d9/zookeeper_0, clientPort=56022, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/cluster_c9d24771-f798-4b7f-5e55-10c183a2c0d9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/cluster_c9d24771-f798-4b7f-5e55-10c183a2c0d9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2023-11-12T09:21:29,314 INFO [Listener at localhost/45829 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=56022 2023-11-12T09:21:29,322 INFO [Listener at localhost/45829 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:21:29,324 INFO [Listener at localhost/45829 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:21:29,988 INFO [Listener at localhost/45829 {}] util.FSUtils(462): Created version file at hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe with version=8 2023-11-12T09:21:29,988 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/hbase-staging 2023-11-12T09:21:30,006 DEBUG [Listener at localhost/45829 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-11-12T09:21:30,006 DEBUG [Listener at localhost/45829 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-11-12T09:21:30,006 DEBUG [Listener at localhost/45829 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-11-12T09:21:30,006 DEBUG [Listener at localhost/45829 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-11-12T09:21:30,221 DEBUG [Listener at localhost/45829 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-11-12T09:21:30,991 INFO [Listener at localhost/45829 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=3 2023-11-12T09:21:31,039 INFO [Listener at localhost/45829 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:21:31,040 INFO [Listener at localhost/45829 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-12T09:21:31,040 INFO [Listener at localhost/45829 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-12T09:21:31,040 INFO [Listener at localhost/45829 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:21:31,040 INFO [Listener at localhost/45829 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-12T09:21:31,202 INFO [Listener at localhost/45829 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-12T09:21:31,345 INFO [Listener at localhost/45829 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-11-12T09:21:31,366 DEBUG [Listener at localhost/45829 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-11-12T09:21:31,372 INFO [Listener at localhost/45829 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-12T09:21:31,421 DEBUG [Listener at localhost/45829 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 29101 (auto-detected) 2023-11-12T09:21:31,423 DEBUG [Listener at localhost/45829 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:0f:ff:fe:94:aa:bf (auto-detected) 2023-11-12T09:21:31,467 INFO [Listener at localhost/45829 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:33463 2023-11-12T09:21:31,482 INFO [Listener at localhost/45829 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:21:31,484 INFO [Listener at localhost/45829 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:21:31,506 INFO [Listener at localhost/45829 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:33463 connecting to ZooKeeper ensemble=127.0.0.1:56022 2023-11-12T09:21:31,567 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:334630x0, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-12T09:21:31,571 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:33463-0x1004ceff8aa0000 connected 2023-11-12T09:21:31,604 DEBUG [Listener at localhost/45829 {}] zookeeper.ZKUtil(113): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-12T09:21:31,607 DEBUG [Listener at localhost/45829 {}] zookeeper.ZKUtil(113): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:21:31,611 DEBUG [Listener at localhost/45829 {}] zookeeper.ZKUtil(113): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-12T09:21:31,622 DEBUG [Listener at localhost/45829 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33463 2023-11-12T09:21:31,622 DEBUG [Listener at localhost/45829 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33463 2023-11-12T09:21:31,622 DEBUG [Listener at localhost/45829 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33463 2023-11-12T09:21:31,623 DEBUG [Listener at localhost/45829 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33463 2023-11-12T09:21:31,623 DEBUG [Listener at localhost/45829 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33463 2023-11-12T09:21:31,677 INFO [Listener at localhost/45829 {}] log.Log(170): Logging initialized @7086ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-11-12T09:21:31,802 INFO [Listener at localhost/45829 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-12T09:21:31,803 INFO [Listener at localhost/45829 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-12T09:21:31,804 INFO [Listener at localhost/45829 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-12T09:21:31,806 INFO [Listener at localhost/45829 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-11-12T09:21:31,806 INFO [Listener at localhost/45829 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-12T09:21:31,806 INFO [Listener at localhost/45829 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-12T09:21:31,809 INFO [Listener at localhost/45829 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-12T09:21:31,938 INFO [Listener at localhost/45829 {}] http.HttpServer(1227): Jetty bound to port 42595 2023-11-12T09:21:31,940 INFO [Listener at localhost/45829 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:21:31,985 INFO [Listener at localhost/45829 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:21:31,990 INFO [Listener at localhost/45829 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@58a5effe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/hadoop.log.dir/,AVAILABLE} 2023-11-12T09:21:31,991 INFO [Listener at localhost/45829 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:21:31,992 INFO [Listener at localhost/45829 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5a55de6b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-12T09:21:32,058 INFO [Listener at localhost/45829 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-12T09:21:32,072 INFO [Listener at localhost/45829 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-12T09:21:32,073 INFO [Listener at localhost/45829 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-12T09:21:32,075 INFO [Listener at localhost/45829 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-11-12T09:21:32,083 INFO [Listener at localhost/45829 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:21:33,295 INFO [Listener at localhost/45829 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@36f4baa6{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-12T09:21:33,306 INFO [Listener at localhost/45829 {}] server.AbstractConnector(333): Started ServerConnector@72c7f512{HTTP/1.1, (http/1.1)}{0.0.0.0:42595} 2023-11-12T09:21:33,306 INFO [Listener at localhost/45829 {}] server.Server(415): Started @8715ms 2023-11-12T09:21:33,310 INFO [Listener at localhost/45829 {}] master.HMaster(486): hbase.rootdir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe, hbase.cluster.distributed=false 2023-11-12T09:21:33,378 INFO [Listener at localhost/45829 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=3 2023-11-12T09:21:33,378 INFO [Listener at localhost/45829 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:21:33,378 INFO [Listener at localhost/45829 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-12T09:21:33,379 INFO [Listener at localhost/45829 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-12T09:21:33,379 INFO [Listener at localhost/45829 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-12T09:21:33,379 INFO [Listener at localhost/45829 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-12T09:21:33,391 INFO [Listener at localhost/45829 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-12T09:21:33,393 INFO [Listener at localhost/45829 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-12T09:21:33,394 INFO [Listener at localhost/45829 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:44111 2023-11-12T09:21:33,397 INFO [Listener at localhost/45829 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-11-12T09:21:33,404 DEBUG [Listener at localhost/45829 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-11-12T09:21:33,405 INFO [Listener at localhost/45829 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:21:33,408 INFO [Listener at localhost/45829 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:21:33,411 INFO [Listener at localhost/45829 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:44111 connecting to ZooKeeper ensemble=127.0.0.1:56022 2023-11-12T09:21:33,414 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:441110x0, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-12T09:21:33,415 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:44111-0x1004ceff8aa0001 connected 2023-11-12T09:21:33,415 DEBUG [Listener at localhost/45829 {}] zookeeper.ZKUtil(113): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-12T09:21:33,417 DEBUG [Listener at localhost/45829 {}] zookeeper.ZKUtil(113): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-12T09:21:33,418 DEBUG [Listener at localhost/45829 {}] zookeeper.ZKUtil(113): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-12T09:21:33,418 DEBUG [Listener at localhost/45829 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44111 2023-11-12T09:21:33,418 DEBUG [Listener at localhost/45829 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44111 2023-11-12T09:21:33,419 DEBUG [Listener at localhost/45829 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44111 2023-11-12T09:21:33,420 DEBUG [Listener at localhost/45829 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44111 2023-11-12T09:21:33,421 DEBUG [Listener at localhost/45829 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44111 2023-11-12T09:21:33,425 INFO [Listener at localhost/45829 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-12T09:21:33,425 INFO [Listener at localhost/45829 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-12T09:21:33,426 INFO [Listener at localhost/45829 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-12T09:21:33,427 INFO [Listener at localhost/45829 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-11-12T09:21:33,427 INFO [Listener at localhost/45829 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-12T09:21:33,427 INFO [Listener at localhost/45829 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-12T09:21:33,427 INFO [Listener at localhost/45829 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-12T09:21:33,430 INFO [Listener at localhost/45829 {}] http.HttpServer(1227): Jetty bound to port 42963 2023-11-12T09:21:33,430 INFO [Listener at localhost/45829 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:21:33,432 INFO [Listener at localhost/45829 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:21:33,432 INFO [Listener at localhost/45829 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2e744413{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/hadoop.log.dir/,AVAILABLE} 2023-11-12T09:21:33,433 INFO [Listener at localhost/45829 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:21:33,433 INFO [Listener at localhost/45829 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3a055401{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-12T09:21:33,447 INFO [Listener at localhost/45829 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-12T09:21:33,449 INFO [Listener at localhost/45829 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-12T09:21:33,449 INFO [Listener at localhost/45829 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-12T09:21:33,449 INFO [Listener at localhost/45829 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-11-12T09:21:33,450 INFO [Listener at localhost/45829 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-12T09:21:33,455 INFO [Listener at localhost/45829 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@7d3ea42b{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-12T09:21:33,457 INFO [Listener at localhost/45829 {}] server.AbstractConnector(333): Started ServerConnector@7c1f26d7{HTTP/1.1, (http/1.1)}{0.0.0.0:42963} 2023-11-12T09:21:33,457 INFO [Listener at localhost/45829 {}] server.Server(415): Started @8866ms 2023-11-12T09:21:33,462 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-12T09:21:33,466 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@6fae22d8{HTTP/1.1, (http/1.1)}{0.0.0.0:33359} 2023-11-12T09:21:33,466 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @8875ms 2023-11-12T09:21:33,466 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2389): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,33463,1699780890205 2023-11-12T09:21:33,508 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:21:33,508 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:21:33,511 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,33463,1699780890205 2023-11-12T09:21:33,516 DEBUG [M:0;jenkins-hbase3:33463 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:33463 2023-11-12T09:21:33,531 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-12T09:21:33,531 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-12T09:21:33,531 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:33,531 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:33,532 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-12T09:21:33,534 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,33463,1699780890205 from backup master directory 2023-11-12T09:21:33,534 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-12T09:21:33,539 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,33463,1699780890205 2023-11-12T09:21:33,539 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:21:33,539 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-12T09:21:33,540 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-12T09:21:33,540 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,33463,1699780890205 2023-11-12T09:21:33,543 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-11-12T09:21:33,545 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-11-12T09:21:33,625 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/hbase.id with ID: a69f1135-cbe4-4ee3-8d4e-102059593744 2023-11-12T09:21:33,669 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-12T09:21:33,687 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:33,687 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:33,737 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:21:33,740 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-11-12T09:21:33,763 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-11-12T09:21:33,763 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-11-12T09:21:33,764 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:21:33,777 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:21:33,780 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:21:33,853 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store 2023-11-12T09:21:33,877 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-11-12T09:21:33,877 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:33,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-12T09:21:33,878 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:21:33,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:21:33,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-12T09:21:33,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:21:33,878 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:21:33,878 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:21:33,880 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/.initializing 2023-11-12T09:21:33,880 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/WALs/jenkins-hbase3.apache.org,33463,1699780890205 2023-11-12T09:21:33,889 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-12T09:21:33,902 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C33463%2C1699780890205, suffix=, logDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/WALs/jenkins-hbase3.apache.org,33463,1699780890205, archiveDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/oldWALs, maxLogs=10 2023-11-12T09:21:33,935 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/WALs/jenkins-hbase3.apache.org,33463,1699780890205/jenkins-hbase3.apache.org%2C33463%2C1699780890205.1699780893909, exclude list is [], retry=0 2023-11-12T09:21:33,971 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK] 2023-11-12T09:21:33,979 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:499) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-12T09:21:34,049 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/WALs/jenkins-hbase3.apache.org,33463,1699780890205/jenkins-hbase3.apache.org%2C33463%2C1699780890205.1699780893909 2023-11-12T09:21:34,050 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK]] 2023-11-12T09:21:34,050 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:21:34,051 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:34,055 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:21:34,056 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:21:34,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:21:34,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-11-12T09:21:34,166 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:34,169 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:21:34,170 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:21:34,173 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-11-12T09:21:34,173 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:34,175 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:21:34,175 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:21:34,178 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-11-12T09:21:34,178 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:34,179 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:21:34,179 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:21:34,182 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-11-12T09:21:34,182 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:34,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:21:34,189 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:21:34,190 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:21:34,205 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-11-12T09:21:34,209 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-11-12T09:21:34,213 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:21:34,214 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11496158560, jitterRate=0.0706631988286972}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-11-12T09:21:34,220 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:21:34,221 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-11-12T09:21:34,256 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c2b635d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:21:34,310 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(861): No meta location available on zookeeper, skip migrating... 2023-11-12T09:21:34,331 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-11-12T09:21:34,332 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-11-12T09:21:34,335 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-11-12T09:21:34,336 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-11-12T09:21:34,343 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 6 msec 2023-11-12T09:21:34,343 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-11-12T09:21:34,379 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-11-12T09:21:34,396 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-11-12T09:21:34,399 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-11-12T09:21:34,402 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-11-12T09:21:34,403 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-11-12T09:21:34,405 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-11-12T09:21:34,408 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-11-12T09:21:34,415 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-11-12T09:21:34,417 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-11-12T09:21:34,419 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-11-12T09:21:34,420 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-11-12T09:21:34,435 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-11-12T09:21:34,437 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-11-12T09:21:34,443 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-12T09:21:34,443 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-12T09:21:34,443 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:34,443 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:34,444 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(805): Active/primary master=jenkins-hbase3.apache.org,33463,1699780890205, sessionid=0x1004ceff8aa0000, setting cluster-up flag (Was=false) 2023-11-12T09:21:34,453 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-11-12T09:21:34,464 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:34,464 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:34,471 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-11-12T09:21:34,472 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33463,1699780890205 2023-11-12T09:21:34,476 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:34,476 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:34,483 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-11-12T09:21:34,484 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33463,1699780890205 2023-11-12T09:21:34,490 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:34,490 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:34,496 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-11-12T09:21:34,497 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,33463,1699780890205 2023-11-12T09:21:34,555 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver loaded, priority=536870911. 2023-11-12T09:21:34,571 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:44111 2023-11-12T09:21:34,573 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(997): ClusterId : a69f1135-cbe4-4ee3-8d4e-102059593744 2023-11-12T09:21:34,575 INFO [RS:0;jenkins-hbase3:44111 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-11-12T09:21:34,579 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-11-12T09:21:34,585 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-11-12T09:21:34,585 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-11-12T09:21:34,589 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-11-12T09:21:34,589 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-11-12T09:21:34,592 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-11-12T09:21:34,593 DEBUG [RS:0;jenkins-hbase3:44111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a3c9459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:21:34,595 DEBUG [RS:0;jenkins-hbase3:44111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a916a3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-11-12T09:21:34,600 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-11-12T09:21:34,600 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-11-12T09:21:34,600 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-11-12T09:21:34,604 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,33463,1699780890205 with isa=jenkins-hbase3.apache.org/172.31.12.81:44111, startcode=1699780893376 2023-11-12T09:21:34,625 DEBUG [RS:0;jenkins-hbase3:44111 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-12T09:21:34,646 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-11-12T09:21:34,666 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-11-12T09:21:34,674 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-11-12T09:21:34,688 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,33463,1699780890205 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-11-12T09:21:34,694 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:21:34,694 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:21:34,695 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:21:34,695 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-11-12T09:21:34,695 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-11-12T09:21:34,695 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:21:34,695 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-11-12T09:21:34,695 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:21:34,706 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1699780924705 2023-11-12T09:21:34,709 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:58741, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-12T09:21:34,710 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-11-12T09:21:34,712 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-11-12T09:21:34,713 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-11-12T09:21:34,715 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-11-12T09:21:34,717 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:34,718 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-12T09:21:34,721 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33463 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3212) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:592) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-12T09:21:34,732 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-11-12T09:21:34,734 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-11-12T09:21:34,734 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-11-12T09:21:34,735 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-11-12T09:21:34,735 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-11-12T09:21:34,736 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:34,737 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-11-12T09:21:34,739 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-11-12T09:21:34,740 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-11-12T09:21:34,747 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x63434533 to 127.0.0.1:56022 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:21:34,757 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c53211, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:21:34,757 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-12T09:21:34,761 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-11-12T09:21:34,761 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-11-12T09:21:34,762 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-11-12T09:21:34,762 WARN [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-11-12T09:21:34,763 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699780894763,5,FailOnTimeoutGroup] 2023-11-12T09:21:34,763 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699780894763,5,FailOnTimeoutGroup] 2023-11-12T09:21:34,763 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:34,763 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1636): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-11-12T09:21:34,765 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:34,765 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:34,863 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,33463,1699780890205 with isa=jenkins-hbase3.apache.org/172.31.12.81:44111, startcode=1699780893376 2023-11-12T09:21:34,868 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33463 {}] master.ServerManager(396): Registering regionserver=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:34,879 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe 2023-11-12T09:21:34,879 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:37959 2023-11-12T09:21:34,879 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=42595 2023-11-12T09:21:34,889 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-11-12T09:21:34,889 DEBUG [RS:0;jenkins-hbase3:44111 {}] zookeeper.ZKUtil(111): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:34,889 WARN [RS:0;jenkins-hbase3:44111 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-12T09:21:34,889 INFO [RS:0;jenkins-hbase3:44111 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:21:34,890 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:34,892 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,44111,1699780893376] 2023-11-12T09:21:34,907 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-11-12T09:21:34,925 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-11-12T09:21:34,939 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-11-12T09:21:34,944 INFO [RS:0;jenkins-hbase3:44111 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-11-12T09:21:34,944 INFO [RS:0;jenkins-hbase3:44111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:34,945 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-11-12T09:21:34,955 INFO [RS:0;jenkins-hbase3:44111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:34,955 DEBUG [RS:0;jenkins-hbase3:44111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:21:34,955 DEBUG [RS:0;jenkins-hbase3:44111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:21:34,956 DEBUG [RS:0;jenkins-hbase3:44111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:21:34,956 DEBUG [RS:0;jenkins-hbase3:44111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:21:34,956 DEBUG [RS:0;jenkins-hbase3:44111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:21:34,956 DEBUG [RS:0;jenkins-hbase3:44111 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-11-12T09:21:34,956 DEBUG [RS:0;jenkins-hbase3:44111 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:21:34,957 DEBUG [RS:0;jenkins-hbase3:44111 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:21:34,957 DEBUG [RS:0;jenkins-hbase3:44111 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:21:34,957 DEBUG [RS:0;jenkins-hbase3:44111 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:21:34,957 DEBUG [RS:0;jenkins-hbase3:44111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-11-12T09:21:34,958 DEBUG [RS:0;jenkins-hbase3:44111 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-11-12T09:21:34,958 DEBUG [RS:0;jenkins-hbase3:44111 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-11-12T09:21:34,959 INFO [RS:0;jenkins-hbase3:44111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:34,959 INFO [RS:0;jenkins-hbase3:44111 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:34,960 INFO [RS:0;jenkins-hbase3:44111 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:34,960 INFO [RS:0;jenkins-hbase3:44111 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:34,960 INFO [RS:0;jenkins-hbase3:44111 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44111,1699780893376-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-12T09:21:34,976 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-11-12T09:21:34,979 INFO [RS:0;jenkins-hbase3:44111 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,44111,1699780893376-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:34,997 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,44111,1699780893376 started 2023-11-12T09:21:34,997 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,44111,1699780893376, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:44111, sessionid=0x1004ceff8aa0001 2023-11-12T09:21:34,997 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-11-12T09:21:34,997 DEBUG [RS:0;jenkins-hbase3:44111 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:34,997 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44111,1699780893376' 2023-11-12T09:21:34,998 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-11-12T09:21:34,998 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-11-12T09:21:34,999 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-11-12T09:21:34,999 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-11-12T09:21:34,999 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44111,1699780893376' 2023-11-12T09:21:34,999 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:21:35,000 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:21:35,000 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-11-12T09:21:35,000 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-11-12T09:21:35,000 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-11-12T09:21:35,000 DEBUG [RS:0;jenkins-hbase3:44111 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:35,001 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,44111,1699780893376' 2023-11-12T09:21:35,001 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-11-12T09:21:35,001 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-11-12T09:21:35,002 DEBUG [RS:0;jenkins-hbase3:44111 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-11-12T09:21:35,002 INFO [RS:0;jenkins-hbase3:44111 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-11-12T09:21:35,002 INFO [RS:0;jenkins-hbase3:44111 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-11-12T09:21:35,115 INFO [RS:0;jenkins-hbase3:44111 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-12T09:21:35,118 INFO [RS:0;jenkins-hbase3:44111 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C44111%2C1699780893376, suffix=, logDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376, archiveDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/oldWALs, maxLogs=32 2023-11-12T09:21:35,130 DEBUG [RS:0;jenkins-hbase3:44111 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780895119, exclude list is [], retry=0 2023-11-12T09:21:35,136 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK] 2023-11-12T09:21:35,137 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-11-12T09:21:35,137 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe 2023-11-12T09:21:35,141 INFO [RS:0;jenkins-hbase3:44111 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780895119 2023-11-12T09:21:35,141 DEBUG [RS:0;jenkins-hbase3:44111 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK]] 2023-11-12T09:21:35,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:35,156 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-12T09:21:35,158 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-12T09:21:35,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:35,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:21:35,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-12T09:21:35,161 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-12T09:21:35,161 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:35,162 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:21:35,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-12T09:21:35,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-12T09:21:35,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:35,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:21:35,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740 2023-11-12T09:21:35,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740 2023-11-12T09:21:35,170 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-12T09:21:35,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-12T09:21:35,175 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:21:35,176 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11028010240, jitterRate=0.027063488960266113}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-12T09:21:35,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-12T09:21:35,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-12T09:21:35,178 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-12T09:21:35,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-12T09:21:35,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-12T09:21:35,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-12T09:21:35,180 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-12T09:21:35,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-12T09:21:35,183 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-11-12T09:21:35,183 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-11-12T09:21:35,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-11-12T09:21:35,204 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-11-12T09:21:35,208 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-11-12T09:21:35,360 DEBUG [jenkins-hbase3:33463 {}] assignment.AssignmentManager(2319): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-11-12T09:21:35,371 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:35,377 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,44111,1699780893376, state=OPENING 2023-11-12T09:21:35,387 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-11-12T09:21:35,388 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:35,388 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:35,389 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:21:35,389 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:21:35,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,44111,1699780893376}] 2023-11-12T09:21:35,713 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:35,716 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-11-12T09:21:35,719 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:52470, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-11-12T09:21:35,731 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-11-12T09:21:35,732 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-12T09:21:35,733 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-11-12T09:21:35,736 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta, suffix=.meta, logDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376, archiveDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/oldWALs, maxLogs=32 2023-11-12T09:21:35,748 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780895737.meta, exclude list is [], retry=0 2023-11-12T09:21:35,752 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK] 2023-11-12T09:21:35,756 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780895737.meta 2023-11-12T09:21:35,756 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK]] 2023-11-12T09:21:35,757 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:21:35,761 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:21:35,762 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-11-12T09:21:35,780 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-11-12T09:21:35,780 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-11-12T09:21:35,785 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-11-12T09:21:35,785 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:35,785 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-11-12T09:21:35,785 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-11-12T09:21:35,788 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-12T09:21:35,789 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-12T09:21:35,790 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:35,790 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:21:35,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-12T09:21:35,792 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-12T09:21:35,792 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:35,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:21:35,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-12T09:21:35,794 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-12T09:21:35,794 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:35,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-12T09:21:35,796 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740 2023-11-12T09:21:35,798 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740 2023-11-12T09:21:35,801 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-12T09:21:35,803 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-12T09:21:35,804 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10131758560, jitterRate=-0.056406453251838684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-12T09:21:35,805 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-12T09:21:35,817 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1699780895704 2023-11-12T09:21:35,837 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:35,838 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,44111,1699780893376, state=OPEN 2023-11-12T09:21:35,841 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-12T09:21:35,841 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-12T09:21:35,842 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:21:35,842 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-12T09:21:35,844 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-11-12T09:21:35,844 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-11-12T09:21:35,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-11-12T09:21:35,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,44111,1699780893376 in 450 msec 2023-11-12T09:21:35,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-11-12T09:21:35,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 658 msec 2023-11-12T09:21:35,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.3020 sec 2023-11-12T09:21:35,867 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1064): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1699780895866, completionTime=-1 2023-11-12T09:21:35,867 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(816): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-11-12T09:21:35,867 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1645): Joining cluster... 2023-11-12T09:21:35,946 DEBUG [hconnection-0x1c987ed5-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:21:35,951 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:52480, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:21:35,976 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1657): Number of RegionServers=1 2023-11-12T09:21:35,976 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1699780955976 2023-11-12T09:21:35,976 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1699781015976 2023-11-12T09:21:35,977 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1664): Joined the cluster in 109 msec 2023-11-12T09:21:36,007 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33463,1699780890205-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:36,007 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33463,1699780890205-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:36,008 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33463,1699780890205-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:36,010 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:33463, period=300000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:36,011 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-11-12T09:21:36,020 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-11-12T09:21:36,034 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-11-12T09:21:36,036 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2369): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-12T09:21:36,047 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-11-12T09:21:36,051 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:21:36,053 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:36,055 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:21:36,480 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 23ec3c63d01117206e0be8e22550ef70, NAME => 'hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe 2023-11-12T09:21:36,894 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:36,895 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 23ec3c63d01117206e0be8e22550ef70, disabling compactions & flushes 2023-11-12T09:21:36,895 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. 2023-11-12T09:21:36,895 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. 2023-11-12T09:21:36,895 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. after waiting 0 ms 2023-11-12T09:21:36,895 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. 2023-11-12T09:21:36,895 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. 2023-11-12T09:21:36,895 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 23ec3c63d01117206e0be8e22550ef70: 2023-11-12T09:21:36,898 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:21:36,912 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1699780896900"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699780896900"}]},"ts":"1699780896900"} 2023-11-12T09:21:36,947 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:21:36,949 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:21:36,957 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780896949"}]},"ts":"1699780896949"} 2023-11-12T09:21:36,961 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-11-12T09:21:36,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=23ec3c63d01117206e0be8e22550ef70, ASSIGN}] 2023-11-12T09:21:36,974 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=23ec3c63d01117206e0be8e22550ef70, ASSIGN 2023-11-12T09:21:36,975 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=23ec3c63d01117206e0be8e22550ef70, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44111,1699780893376; forceNewPlan=false, retain=false 2023-11-12T09:21:37,126 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=23ec3c63d01117206e0be8e22550ef70, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:37,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 23ec3c63d01117206e0be8e22550ef70, server=jenkins-hbase3.apache.org,44111,1699780893376}] 2023-11-12T09:21:37,290 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:37,299 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. 2023-11-12T09:21:37,300 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 23ec3c63d01117206e0be8e22550ef70, NAME => 'hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:21:37,301 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:21:37,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 23ec3c63d01117206e0be8e22550ef70 2023-11-12T09:21:37,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:37,302 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 23ec3c63d01117206e0be8e22550ef70 2023-11-12T09:21:37,302 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 23ec3c63d01117206e0be8e22550ef70 2023-11-12T09:21:37,304 INFO [StoreOpener-23ec3c63d01117206e0be8e22550ef70-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 23ec3c63d01117206e0be8e22550ef70 2023-11-12T09:21:37,306 INFO [StoreOpener-23ec3c63d01117206e0be8e22550ef70-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23ec3c63d01117206e0be8e22550ef70 columnFamilyName info 2023-11-12T09:21:37,306 DEBUG [StoreOpener-23ec3c63d01117206e0be8e22550ef70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:37,306 INFO [StoreOpener-23ec3c63d01117206e0be8e22550ef70-1 {}] regionserver.HStore(324): Store=23ec3c63d01117206e0be8e22550ef70/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:21:37,308 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/namespace/23ec3c63d01117206e0be8e22550ef70 2023-11-12T09:21:37,309 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/namespace/23ec3c63d01117206e0be8e22550ef70 2023-11-12T09:21:37,312 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 23ec3c63d01117206e0be8e22550ef70 2023-11-12T09:21:37,320 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/namespace/23ec3c63d01117206e0be8e22550ef70/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:21:37,321 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 23ec3c63d01117206e0be8e22550ef70; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11033937280, jitterRate=0.027615487575531006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:21:37,323 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 23ec3c63d01117206e0be8e22550ef70: 2023-11-12T09:21:37,325 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70., pid=6, masterSystemTime=1699780897290 2023-11-12T09:21:37,330 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. 2023-11-12T09:21:37,330 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. 2023-11-12T09:21:37,331 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=23ec3c63d01117206e0be8e22550ef70, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:37,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-11-12T09:21:37,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 23ec3c63d01117206e0be8e22550ef70, server=jenkins-hbase3.apache.org,44111,1699780893376 in 200 msec 2023-11-12T09:21:37,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-11-12T09:21:37,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=23ec3c63d01117206e0be8e22550ef70, ASSIGN in 368 msec 2023-11-12T09:21:37,343 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:21:37,343 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780897343"}]},"ts":"1699780897343"} 2023-11-12T09:21:37,346 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-11-12T09:21:37,350 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:21:37,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.3130 sec 2023-11-12T09:21:37,355 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-11-12T09:21:37,356 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-11-12T09:21:37,356 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:37,356 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-12T09:21:37,416 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-11-12T09:21:37,438 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:21:37,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 40 msec 2023-11-12T09:21:37,451 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-11-12T09:21:37,464 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:21:37,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 18 msec 2023-11-12T09:21:37,480 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-11-12T09:21:37,482 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-11-12T09:21:37,483 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1194): Master has completed initialization 3.943sec 2023-11-12T09:21:37,485 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-11-12T09:21:37,487 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-11-12T09:21:37,488 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-11-12T09:21:37,489 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-11-12T09:21:37,489 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-11-12T09:21:37,491 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33463,1699780890205-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-12T09:21:37,491 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,33463,1699780890205-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-11-12T09:21:37,508 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1297): Balancer post startup initialization complete, took 0 seconds 2023-11-12T09:21:37,510 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-11-12T09:21:37,532 DEBUG [Listener at localhost/45829 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2a7653f5 to 127.0.0.1:56022 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:21:37,540 DEBUG [Listener at localhost/45829 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@154a74df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:21:37,545 DEBUG [Listener at localhost/45829 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-11-12T09:21:37,545 DEBUG [Listener at localhost/45829 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-11-12T09:21:37,561 DEBUG [hconnection-0x358f80a6-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:21:37,575 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:52490, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:21:37,594 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,33463,1699780890205 2023-11-12T09:21:37,595 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-11-12T09:21:37,595 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/test.cache.data in system properties and HBase conf 2023-11-12T09:21:37,595 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/hadoop.tmp.dir in system properties and HBase conf 2023-11-12T09:21:37,595 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/hadoop.log.dir in system properties and HBase conf 2023-11-12T09:21:37,595 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-12T09:21:37,595 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-12T09:21:37,596 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-12T09:21:37,596 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:21:37,596 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-12T09:21:37,596 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-12T09:21:37,596 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:21:37,596 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-12T09:21:37,597 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-12T09:21:37,597 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-12T09:21:37,597 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:21:37,597 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-12T09:21:37,597 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/nfs.dump.dir in system properties and HBase conf 2023-11-12T09:21:37,597 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/java.io.tmpdir in system properties and HBase conf 2023-11-12T09:21:37,597 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-12T09:21:37,597 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-12T09:21:37,598 INFO [Listener at localhost/45829 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-11-12T09:21:38,770 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:21:39,201 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:21:39,207 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_34469_jobhistory____dshbr4/webapp 2023-11-12T09:21:39,373 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-11-12T09:21:39,373 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-11-12T09:21:39,373 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:21:39,377 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:21:39,466 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:21:39,628 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:21:39,976 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-11-12T09:21:40,010 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34469 2023-11-12T09:21:40,267 WARN [Listener at jenkins-hbase3.apache.org/43745 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:21:40,353 WARN [Listener at jenkins-hbase3.apache.org/43745 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:21:40,354 INFO [Listener at jenkins-hbase3.apache.org/43745 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:21:40,360 INFO [Listener at jenkins-hbase3.apache.org/43745 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_38213_cluster____61s6lw/webapp 2023-11-12T09:21:40,438 INFO [Listener at jenkins-hbase3.apache.org/43745 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:21:40,438 INFO [Listener at jenkins-hbase3.apache.org/43745 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-11-12T09:21:40,438 INFO [Listener at jenkins-hbase3.apache.org/43745 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:21:40,439 INFO [Listener at jenkins-hbase3.apache.org/43745 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:21:40,454 INFO [Listener at jenkins-hbase3.apache.org/43745 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:21:40,710 INFO [Listener at jenkins-hbase3.apache.org/43745 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:21:40,843 INFO [Listener at jenkins-hbase3.apache.org/43745 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:21:40,855 INFO [Listener at jenkins-hbase3.apache.org/43745 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:38213 2023-11-12T09:21:41,220 WARN [Listener at jenkins-hbase3.apache.org/36701 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:21:41,246 WARN [Listener at jenkins-hbase3.apache.org/36701 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:21:41,247 INFO [Listener at jenkins-hbase3.apache.org/36701 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:21:41,257 INFO [Listener at jenkins-hbase3.apache.org/36701 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_45359_node____.xauw2z/webapp 2023-11-12T09:21:41,324 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:21:41,328 INFO [Listener at jenkins-hbase3.apache.org/36701 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-12T09:21:41,328 INFO [Listener at jenkins-hbase3.apache.org/36701 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:21:41,329 INFO [Listener at jenkins-hbase3.apache.org/36701 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:21:41,329 INFO [Listener at jenkins-hbase3.apache.org/36701 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:21:41,347 INFO [Listener at jenkins-hbase3.apache.org/36701 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:21:41,384 INFO [Listener at jenkins-hbase3.apache.org/36701 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:21:41,408 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-11-12T09:21:41,409 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-11-12T09:21:41,410 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-11-12T09:21:41,411 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-11-12T09:21:41,413 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-11-12T09:21:41,413 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-11-12T09:21:41,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: Master,sub=Coprocessor.Master.CP_org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver 2023-11-12T09:21:41,415 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering Master,sub=Coprocessor.Master.CP_org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver Metrics about HBase MasterObservers 2023-11-12T09:21:41,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:21:41,415 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-11-12T09:21:41,416 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-12T09:21:41,416 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-11-12T09:21:41,416 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-11-12T09:21:41,416 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-11-12T09:21:41,474 INFO [Listener at jenkins-hbase3.apache.org/36701 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:21:41,484 INFO [Listener at jenkins-hbase3.apache.org/36701 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:45359 2023-11-12T09:21:41,657 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-12T09:21:41,664 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-12T09:21:41,664 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-12T09:21:41,672 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_44191_node____.xzjom0/webapp 2023-11-12T09:21:41,723 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-12T09:21:41,723 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-12T09:21:41,723 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-12T09:21:41,723 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-12T09:21:41,731 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:21:41,747 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:21:41,807 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-12T09:21:41,813 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:44191 2023-11-12T09:21:41,850 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-11-12T09:21:41,850 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:37959/backupUT 2023-11-12T09:21:41,862 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:21:41,866 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:52850, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:21:41,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-11-12T09:21:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-11-12T09:21:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-12T09:21:41,904 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:21:41,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 32 msec 2023-11-12T09:21:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-12T09:21:42,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-11-12T09:21:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-11-12T09:21:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-12T09:21:42,017 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:21:42,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 21 msec 2023-11-12T09:21:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-12T09:21:42,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-11-12T09:21:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-11-12T09:21:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-12T09:21:42,127 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:21:42,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 18 msec 2023-11-12T09:21:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-12T09:21:42,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-11-12T09:21:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-11-12T09:21:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-12T09:21:42,239 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:21:42,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 18 msec 2023-11-12T09:21:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-12T09:21:42,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'test-1699780901850', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:21:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1699780901850 2023-11-12T09:21:42,353 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1699780901850 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:21:42,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1699780901850" procId is: 13 2023-11-12T09:21:42,353 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:42,355 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1699780901850 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:21:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:21:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:21:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:21:42,799 INFO [RegionOpenAndInit-test-1699780901850-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9d3fa1e74299c566cf2ed2c7ca01ec35, NAME => 'test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1699780901850', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe 2023-11-12T09:21:42,816 DEBUG [RegionOpenAndInit-test-1699780901850-pool-0 {}] regionserver.HRegion(888): Instantiated test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:42,816 DEBUG [RegionOpenAndInit-test-1699780901850-pool-0 {}] regionserver.HRegion(1668): Closing 9d3fa1e74299c566cf2ed2c7ca01ec35, disabling compactions & flushes 2023-11-12T09:21:42,816 INFO [RegionOpenAndInit-test-1699780901850-pool-0 {}] regionserver.HRegion(1690): Closing region test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:21:42,816 DEBUG [RegionOpenAndInit-test-1699780901850-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:21:42,816 DEBUG [RegionOpenAndInit-test-1699780901850-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. after waiting 0 ms 2023-11-12T09:21:42,816 DEBUG [RegionOpenAndInit-test-1699780901850-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:21:42,816 INFO [RegionOpenAndInit-test-1699780901850-pool-0 {}] regionserver.HRegion(1909): Closed test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:21:42,816 DEBUG [RegionOpenAndInit-test-1699780901850-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9d3fa1e74299c566cf2ed2c7ca01ec35: 2023-11-12T09:21:42,819 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1699780901850 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:21:42,819 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1699780902819"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699780902819"}]},"ts":"1699780902819"} 2023-11-12T09:21:42,822 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:21:42,823 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1699780901850 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:21:42,823 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699780901850","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780902823"}]},"ts":"1699780902823"} 2023-11-12T09:21:42,826 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699780901850, state=ENABLING in hbase:meta 2023-11-12T09:21:42,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699780901850, region=9d3fa1e74299c566cf2ed2c7ca01ec35, ASSIGN}] 2023-11-12T09:21:42,832 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1699780901850, region=9d3fa1e74299c566cf2ed2c7ca01ec35, ASSIGN 2023-11-12T09:21:42,834 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1699780901850, region=9d3fa1e74299c566cf2ed2c7ca01ec35, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44111,1699780893376; forceNewPlan=false, retain=false 2023-11-12T09:21:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:21:42,986 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=9d3fa1e74299c566cf2ed2c7ca01ec35, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:42,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35, server=jenkins-hbase3.apache.org,44111,1699780893376}] 2023-11-12T09:21:43,142 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:43,150 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:21:43,150 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 9d3fa1e74299c566cf2ed2c7ca01ec35, NAME => 'test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:21:43,151 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:21:43,151 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1699780901850 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:21:43,151 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:43,151 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:21:43,151 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:21:43,154 INFO [StoreOpener-9d3fa1e74299c566cf2ed2c7ca01ec35-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:21:43,156 INFO [StoreOpener-9d3fa1e74299c566cf2ed2c7ca01ec35-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d3fa1e74299c566cf2ed2c7ca01ec35 columnFamilyName f 2023-11-12T09:21:43,156 DEBUG [StoreOpener-9d3fa1e74299c566cf2ed2c7ca01ec35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:43,158 INFO [StoreOpener-9d3fa1e74299c566cf2ed2c7ca01ec35-1 {}] regionserver.HStore(324): Store=9d3fa1e74299c566cf2ed2c7ca01ec35/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:21:43,160 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:21:43,161 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:21:43,164 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:21:43,170 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:21:43,171 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 9d3fa1e74299c566cf2ed2c7ca01ec35; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9469228640, jitterRate=-0.11810936033725739}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:21:43,172 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 9d3fa1e74299c566cf2ed2c7ca01ec35: 2023-11-12T09:21:43,174 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35., pid=15, masterSystemTime=1699780903142 2023-11-12T09:21:43,177 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:21:43,177 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:21:43,178 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=9d3fa1e74299c566cf2ed2c7ca01ec35, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:43,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-11-12T09:21:43,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35, server=jenkins-hbase3.apache.org,44111,1699780893376 in 191 msec 2023-11-12T09:21:43,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-11-12T09:21:43,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1699780901850, region=9d3fa1e74299c566cf2ed2c7ca01ec35, ASSIGN in 353 msec 2023-11-12T09:21:43,186 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1699780901850 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:21:43,186 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1699780901850","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780903186"}]},"ts":"1699780903186"} 2023-11-12T09:21:43,188 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1699780901850, state=ENABLED in hbase:meta 2023-11-12T09:21:43,191 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1699780901850 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:21:43,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1699780901850 in 844 msec 2023-11-12T09:21:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-12T09:21:43,470 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: default:test-1699780901850, procId: 13 completed 2023-11-12T09:21:43,472 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0eaeb607 to 127.0.0.1:56022 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:21:43,480 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@151b394d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:21:43,482 DEBUG [hconnection-0x19d31be2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:21:43,485 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:52498, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:21:43,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44111 {}] regionserver.HRegion(8210): writing data to region test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. with WAL disabled. Data may be lost in the event of a crash. 2023-11-12T09:21:43,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns2:test-16997809018501', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:21:43,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16997809018501 2023-11-12T09:21:43,601 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16997809018501 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:21:43,601 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:43,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16997809018501" procId is: 16 2023-11-12T09:21:43,602 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16997809018501 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:21:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:21:43,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:21:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:21:44,026 INFO [RegionOpenAndInit-ns2:test-16997809018501-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => fc49678786bc319888817a55d875958b, NAME => 'ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16997809018501', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe 2023-11-12T09:21:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:21:44,442 DEBUG [RegionOpenAndInit-ns2:test-16997809018501-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:44,442 DEBUG [RegionOpenAndInit-ns2:test-16997809018501-pool-0 {}] regionserver.HRegion(1668): Closing fc49678786bc319888817a55d875958b, disabling compactions & flushes 2023-11-12T09:21:44,442 INFO [RegionOpenAndInit-ns2:test-16997809018501-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. 2023-11-12T09:21:44,442 DEBUG [RegionOpenAndInit-ns2:test-16997809018501-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. 2023-11-12T09:21:44,442 DEBUG [RegionOpenAndInit-ns2:test-16997809018501-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. after waiting 0 ms 2023-11-12T09:21:44,442 DEBUG [RegionOpenAndInit-ns2:test-16997809018501-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. 2023-11-12T09:21:44,443 INFO [RegionOpenAndInit-ns2:test-16997809018501-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. 2023-11-12T09:21:44,443 DEBUG [RegionOpenAndInit-ns2:test-16997809018501-pool-0 {}] regionserver.HRegion(1622): Region close journal for fc49678786bc319888817a55d875958b: 2023-11-12T09:21:44,444 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16997809018501 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:21:44,444 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699780904444"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699780904444"}]},"ts":"1699780904444"} 2023-11-12T09:21:44,447 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:21:44,448 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16997809018501 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:21:44,448 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997809018501","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780904448"}]},"ts":"1699780904448"} 2023-11-12T09:21:44,450 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997809018501, state=ENABLING in hbase:meta 2023-11-12T09:21:44,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997809018501, region=fc49678786bc319888817a55d875958b, ASSIGN}] 2023-11-12T09:21:44,458 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16997809018501, region=fc49678786bc319888817a55d875958b, ASSIGN 2023-11-12T09:21:44,460 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16997809018501, region=fc49678786bc319888817a55d875958b, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44111,1699780893376; forceNewPlan=false, retain=false 2023-11-12T09:21:44,611 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=fc49678786bc319888817a55d875958b, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:44,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure fc49678786bc319888817a55d875958b, server=jenkins-hbase3.apache.org,44111,1699780893376}] 2023-11-12T09:21:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:21:44,767 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:44,772 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. 2023-11-12T09:21:44,772 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => fc49678786bc319888817a55d875958b, NAME => 'ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:21:44,773 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:21:44,773 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997809018501 fc49678786bc319888817a55d875958b 2023-11-12T09:21:44,773 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:44,773 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for fc49678786bc319888817a55d875958b 2023-11-12T09:21:44,773 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for fc49678786bc319888817a55d875958b 2023-11-12T09:21:44,775 INFO [StoreOpener-fc49678786bc319888817a55d875958b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region fc49678786bc319888817a55d875958b 2023-11-12T09:21:44,777 INFO [StoreOpener-fc49678786bc319888817a55d875958b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc49678786bc319888817a55d875958b columnFamilyName f 2023-11-12T09:21:44,777 DEBUG [StoreOpener-fc49678786bc319888817a55d875958b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:44,778 INFO [StoreOpener-fc49678786bc319888817a55d875958b-1 {}] regionserver.HStore(324): Store=fc49678786bc319888817a55d875958b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:21:44,779 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns2/test-16997809018501/fc49678786bc319888817a55d875958b 2023-11-12T09:21:44,780 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns2/test-16997809018501/fc49678786bc319888817a55d875958b 2023-11-12T09:21:44,783 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for fc49678786bc319888817a55d875958b 2023-11-12T09:21:44,796 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns2/test-16997809018501/fc49678786bc319888817a55d875958b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:21:44,797 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened fc49678786bc319888817a55d875958b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10539749120, jitterRate=-0.018409371376037598}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:21:44,798 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for fc49678786bc319888817a55d875958b: 2023-11-12T09:21:44,799 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b., pid=18, masterSystemTime=1699780904767 2023-11-12T09:21:44,802 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. 2023-11-12T09:21:44,802 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. 2023-11-12T09:21:44,803 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=fc49678786bc319888817a55d875958b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:44,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-11-12T09:21:44,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure fc49678786bc319888817a55d875958b, server=jenkins-hbase3.apache.org,44111,1699780893376 in 192 msec 2023-11-12T09:21:44,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-11-12T09:21:44,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16997809018501, region=fc49678786bc319888817a55d875958b, ASSIGN in 353 msec 2023-11-12T09:21:44,813 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16997809018501 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:21:44,814 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16997809018501","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780904813"}]},"ts":"1699780904813"} 2023-11-12T09:21:44,816 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16997809018501, state=ENABLED in hbase:meta 2023-11-12T09:21:44,820 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16997809018501 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:21:44,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16997809018501 in 1.2230 sec 2023-11-12T09:21:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-12T09:21:45,714 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns2:test-16997809018501, procId: 16 completed 2023-11-12T09:21:45,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44111 {}] regionserver.HRegion(8210): writing data to region ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. with WAL disabled. Data may be lost in the event of a crash. 2023-11-12T09:21:45,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns3:test-16997809018502', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:21:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16997809018502 2023-11-12T09:21:45,820 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16997809018502 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:21:45,820 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:45,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16997809018502" procId is: 19 2023-11-12T09:21:45,821 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16997809018502 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:21:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:21:45,835 INFO [RegionOpenAndInit-ns3:test-16997809018502-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b74eabb10ae7ceb40fbd1cc11e47491c, NAME => 'ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16997809018502', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe 2023-11-12T09:21:45,850 DEBUG [RegionOpenAndInit-ns3:test-16997809018502-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:45,850 DEBUG [RegionOpenAndInit-ns3:test-16997809018502-pool-0 {}] regionserver.HRegion(1668): Closing b74eabb10ae7ceb40fbd1cc11e47491c, disabling compactions & flushes 2023-11-12T09:21:45,850 INFO [RegionOpenAndInit-ns3:test-16997809018502-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. 2023-11-12T09:21:45,850 DEBUG [RegionOpenAndInit-ns3:test-16997809018502-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. 2023-11-12T09:21:45,850 DEBUG [RegionOpenAndInit-ns3:test-16997809018502-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. after waiting 0 ms 2023-11-12T09:21:45,850 DEBUG [RegionOpenAndInit-ns3:test-16997809018502-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. 2023-11-12T09:21:45,850 INFO [RegionOpenAndInit-ns3:test-16997809018502-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. 2023-11-12T09:21:45,850 DEBUG [RegionOpenAndInit-ns3:test-16997809018502-pool-0 {}] regionserver.HRegion(1622): Region close journal for b74eabb10ae7ceb40fbd1cc11e47491c: 2023-11-12T09:21:45,852 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16997809018502 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:21:45,852 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699780905852"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699780905852"}]},"ts":"1699780905852"} 2023-11-12T09:21:45,854 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:21:45,856 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16997809018502 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:21:45,856 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997809018502","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780905856"}]},"ts":"1699780905856"} 2023-11-12T09:21:45,858 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997809018502, state=ENABLING in hbase:meta 2023-11-12T09:21:45,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997809018502, region=b74eabb10ae7ceb40fbd1cc11e47491c, ASSIGN}] 2023-11-12T09:21:45,867 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16997809018502, region=b74eabb10ae7ceb40fbd1cc11e47491c, ASSIGN 2023-11-12T09:21:45,868 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16997809018502, region=b74eabb10ae7ceb40fbd1cc11e47491c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44111,1699780893376; forceNewPlan=false, retain=false 2023-11-12T09:21:45,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:21:46,020 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=b74eabb10ae7ceb40fbd1cc11e47491c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:46,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure b74eabb10ae7ceb40fbd1cc11e47491c, server=jenkins-hbase3.apache.org,44111,1699780893376}] 2023-11-12T09:21:46,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:21:46,176 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:46,182 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. 2023-11-12T09:21:46,182 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => b74eabb10ae7ceb40fbd1cc11e47491c, NAME => 'ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:21:46,182 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:21:46,182 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997809018502 b74eabb10ae7ceb40fbd1cc11e47491c 2023-11-12T09:21:46,182 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:46,182 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for b74eabb10ae7ceb40fbd1cc11e47491c 2023-11-12T09:21:46,183 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for b74eabb10ae7ceb40fbd1cc11e47491c 2023-11-12T09:21:46,186 INFO [StoreOpener-b74eabb10ae7ceb40fbd1cc11e47491c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region b74eabb10ae7ceb40fbd1cc11e47491c 2023-11-12T09:21:46,188 INFO [StoreOpener-b74eabb10ae7ceb40fbd1cc11e47491c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b74eabb10ae7ceb40fbd1cc11e47491c columnFamilyName f 2023-11-12T09:21:46,188 DEBUG [StoreOpener-b74eabb10ae7ceb40fbd1cc11e47491c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:46,189 INFO [StoreOpener-b74eabb10ae7ceb40fbd1cc11e47491c-1 {}] regionserver.HStore(324): Store=b74eabb10ae7ceb40fbd1cc11e47491c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:21:46,190 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns3/test-16997809018502/b74eabb10ae7ceb40fbd1cc11e47491c 2023-11-12T09:21:46,190 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns3/test-16997809018502/b74eabb10ae7ceb40fbd1cc11e47491c 2023-11-12T09:21:46,194 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for b74eabb10ae7ceb40fbd1cc11e47491c 2023-11-12T09:21:46,197 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns3/test-16997809018502/b74eabb10ae7ceb40fbd1cc11e47491c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:21:46,198 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened b74eabb10ae7ceb40fbd1cc11e47491c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10373510880, jitterRate=-0.03389151394367218}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:21:46,199 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for b74eabb10ae7ceb40fbd1cc11e47491c: 2023-11-12T09:21:46,200 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c., pid=21, masterSystemTime=1699780906176 2023-11-12T09:21:46,203 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. 2023-11-12T09:21:46,203 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. 2023-11-12T09:21:46,204 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=b74eabb10ae7ceb40fbd1cc11e47491c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:46,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-11-12T09:21:46,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure b74eabb10ae7ceb40fbd1cc11e47491c, server=jenkins-hbase3.apache.org,44111,1699780893376 in 183 msec 2023-11-12T09:21:46,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-11-12T09:21:46,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16997809018502, region=b74eabb10ae7ceb40fbd1cc11e47491c, ASSIGN in 344 msec 2023-11-12T09:21:46,212 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16997809018502 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:21:46,212 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16997809018502","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780906212"}]},"ts":"1699780906212"} 2023-11-12T09:21:46,214 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16997809018502, state=ENABLED in hbase:meta 2023-11-12T09:21:46,217 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16997809018502 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:21:46,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16997809018502 in 402 msec 2023-11-12T09:21:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-12T09:21:46,428 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns3:test-16997809018502, procId: 19 completed 2023-11-12T09:21:46,428 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16997809018502 get assigned. Timeout = 60000ms 2023-11-12T09:21:46,430 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:21:46,435 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16997809018502 assigned to meta. Checking AM states. 2023-11-12T09:21:46,436 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:21:46,436 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16997809018502 assigned. 2023-11-12T09:21:46,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'ns4:test-16997809018503', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:21:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16997809018503 2023-11-12T09:21:46,442 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16997809018503 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:21:46,443 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:46,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16997809018503" procId is: 22 2023-11-12T09:21:46,444 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16997809018503 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:21:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:21:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:21:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:21:46,860 INFO [RegionOpenAndInit-ns4:test-16997809018503-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 0c6b0fe5f280595167fe10135a31e3d9, NAME => 'ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16997809018503', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe 2023-11-12T09:21:46,886 DEBUG [RegionOpenAndInit-ns4:test-16997809018503-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:46,886 DEBUG [RegionOpenAndInit-ns4:test-16997809018503-pool-0 {}] regionserver.HRegion(1668): Closing 0c6b0fe5f280595167fe10135a31e3d9, disabling compactions & flushes 2023-11-12T09:21:46,886 INFO [RegionOpenAndInit-ns4:test-16997809018503-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. 2023-11-12T09:21:46,886 DEBUG [RegionOpenAndInit-ns4:test-16997809018503-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. 2023-11-12T09:21:46,886 DEBUG [RegionOpenAndInit-ns4:test-16997809018503-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. after waiting 0 ms 2023-11-12T09:21:46,886 DEBUG [RegionOpenAndInit-ns4:test-16997809018503-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. 2023-11-12T09:21:46,886 INFO [RegionOpenAndInit-ns4:test-16997809018503-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. 2023-11-12T09:21:46,886 DEBUG [RegionOpenAndInit-ns4:test-16997809018503-pool-0 {}] regionserver.HRegion(1622): Region close journal for 0c6b0fe5f280595167fe10135a31e3d9: 2023-11-12T09:21:46,888 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16997809018503 execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:21:46,888 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1699780906888"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699780906888"}]},"ts":"1699780906888"} 2023-11-12T09:21:46,890 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:21:46,892 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16997809018503 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:21:46,892 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16997809018503","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780906892"}]},"ts":"1699780906892"} 2023-11-12T09:21:46,893 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16997809018503, state=ENABLING in hbase:meta 2023-11-12T09:21:46,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16997809018503, region=0c6b0fe5f280595167fe10135a31e3d9, ASSIGN}] 2023-11-12T09:21:46,901 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16997809018503, region=0c6b0fe5f280595167fe10135a31e3d9, ASSIGN 2023-11-12T09:21:46,903 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16997809018503, region=0c6b0fe5f280595167fe10135a31e3d9, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44111,1699780893376; forceNewPlan=false, retain=false 2023-11-12T09:21:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:21:47,054 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=0c6b0fe5f280595167fe10135a31e3d9, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:47,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 0c6b0fe5f280595167fe10135a31e3d9, server=jenkins-hbase3.apache.org,44111,1699780893376}] 2023-11-12T09:21:47,209 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:47,214 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. 2023-11-12T09:21:47,214 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 0c6b0fe5f280595167fe10135a31e3d9, NAME => 'ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:21:47,215 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:21:47,215 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16997809018503 0c6b0fe5f280595167fe10135a31e3d9 2023-11-12T09:21:47,215 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:47,215 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 0c6b0fe5f280595167fe10135a31e3d9 2023-11-12T09:21:47,215 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 0c6b0fe5f280595167fe10135a31e3d9 2023-11-12T09:21:47,218 INFO [StoreOpener-0c6b0fe5f280595167fe10135a31e3d9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 0c6b0fe5f280595167fe10135a31e3d9 2023-11-12T09:21:47,221 INFO [StoreOpener-0c6b0fe5f280595167fe10135a31e3d9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0c6b0fe5f280595167fe10135a31e3d9 columnFamilyName f 2023-11-12T09:21:47,221 DEBUG [StoreOpener-0c6b0fe5f280595167fe10135a31e3d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:47,222 INFO [StoreOpener-0c6b0fe5f280595167fe10135a31e3d9-1 {}] regionserver.HStore(324): Store=0c6b0fe5f280595167fe10135a31e3d9/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:21:47,223 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns4/test-16997809018503/0c6b0fe5f280595167fe10135a31e3d9 2023-11-12T09:21:47,224 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns4/test-16997809018503/0c6b0fe5f280595167fe10135a31e3d9 2023-11-12T09:21:47,227 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 0c6b0fe5f280595167fe10135a31e3d9 2023-11-12T09:21:47,230 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns4/test-16997809018503/0c6b0fe5f280595167fe10135a31e3d9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:21:47,232 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 0c6b0fe5f280595167fe10135a31e3d9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11523304000, jitterRate=0.0731913149356842}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-12T09:21:47,232 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 0c6b0fe5f280595167fe10135a31e3d9: 2023-11-12T09:21:47,233 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9., pid=24, masterSystemTime=1699780907209 2023-11-12T09:21:47,236 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. 2023-11-12T09:21:47,236 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. 2023-11-12T09:21:47,236 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=0c6b0fe5f280595167fe10135a31e3d9, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:47,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-11-12T09:21:47,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 0c6b0fe5f280595167fe10135a31e3d9, server=jenkins-hbase3.apache.org,44111,1699780893376 in 182 msec 2023-11-12T09:21:47,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-11-12T09:21:47,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16997809018503, region=0c6b0fe5f280595167fe10135a31e3d9, ASSIGN in 342 msec 2023-11-12T09:21:47,245 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16997809018503 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:21:47,245 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16997809018503","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780907245"}]},"ts":"1699780907245"} 2023-11-12T09:21:47,247 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16997809018503, state=ENABLED in hbase:meta 2023-11-12T09:21:47,259 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16997809018503 execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:21:47,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16997809018503 in 820 msec 2023-11-12T09:21:47,410 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16997809018501' 2023-11-12T09:21:47,411 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1699780901850' 2023-11-12T09:21:47,411 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16997809018503' 2023-11-12T09:21:47,412 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16997809018502' 2023-11-12T09:21:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-12T09:21:47,553 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns4:test-16997809018503, procId: 22 completed 2023-11-12T09:21:47,553 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16997809018503 get assigned. Timeout = 60000ms 2023-11-12T09:21:47,553 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:21:47,558 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16997809018503 assigned to meta. Checking AM states. 2023-11-12T09:21:47,558 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-12T09:21:47,558 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16997809018503 assigned. 2023-11-12T09:21:47,559 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0eaeb607 to 127.0.0.1:56022 2023-11-12T09:21:47,559 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:21:47,608 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.ResourceChecker(147): before: backup.TestBackupDeleteWithFailures#testBackupDeleteWithFailures Thread=584, OpenFileDescriptor=689, MaxFileDescriptor=60000, SystemLoadAverage=250, ProcessCount=172, AvailableMemoryMB=5846 2023-11-12T09:21:47,608 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.ResourceChecker(130): Thread=584 is superior to 500 2023-11-12T09:21:47,610 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] backup.TestBackupDeleteWithFailures(143): test repair backup delete on a single table with data and failures PRE_DELETE_SNAPSHOT_FAILURE 2023-11-12T09:21:47,612 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3550ce00 to 127.0.0.1:56022 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:21:47,619 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c77358f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:21:47,628 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:21:47,629 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:33916, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:21:47,643 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:21:47,644 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:32864, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:21:47,656 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$18(3398): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-11-12T09:21:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-11-12T09:21:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-12T09:21:47,670 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-12T09:21:47,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 17 msec 2023-11-12T09:21:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-12T09:21:47,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:21:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-11-12T09:21:47,787 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:21:47,787 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:47,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-11-12T09:21:47,788 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:21:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:21:47,808 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => bf8e1ef1f6635e037c32e6a8e41782e2, NAME => 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe 2023-11-12T09:21:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:21:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:21:48,224 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:48,224 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing bf8e1ef1f6635e037c32e6a8e41782e2, disabling compactions & flushes 2023-11-12T09:21:48,224 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:21:48,224 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:21:48,224 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. after waiting 0 ms 2023-11-12T09:21:48,224 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:21:48,224 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:21:48,224 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for bf8e1ef1f6635e037c32e6a8e41782e2: 2023-11-12T09:21:48,226 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:21:48,226 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1699780908226"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699780908226"}]},"ts":"1699780908226"} 2023-11-12T09:21:48,235 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:21:48,236 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:21:48,236 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780908236"}]},"ts":"1699780908236"} 2023-11-12T09:21:48,238 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-12T09:21:48,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, ASSIGN}] 2023-11-12T09:21:48,248 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, ASSIGN 2023-11-12T09:21:48,249 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44111,1699780893376; forceNewPlan=false, retain=false 2023-11-12T09:21:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:21:48,400 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=bf8e1ef1f6635e037c32e6a8e41782e2, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:48,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2, server=jenkins-hbase3.apache.org,44111,1699780893376}] 2023-11-12T09:21:48,555 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:48,560 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:21:48,560 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => bf8e1ef1f6635e037c32e6a8e41782e2, NAME => 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:21:48,561 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:21:48,561 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:21:48,561 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:48,561 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:21:48,561 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:21:48,563 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:21:48,565 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf8e1ef1f6635e037c32e6a8e41782e2 columnFamilyName meta 2023-11-12T09:21:48,566 DEBUG [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:48,566 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.HStore(324): Store=bf8e1ef1f6635e037c32e6a8e41782e2/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:21:48,566 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:21:48,568 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf8e1ef1f6635e037c32e6a8e41782e2 columnFamilyName session 2023-11-12T09:21:48,568 DEBUG [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:48,569 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.HStore(324): Store=bf8e1ef1f6635e037c32e6a8e41782e2/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:21:48,570 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:21:48,570 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:21:48,572 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:21:48,573 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:21:48,576 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:21:48,576 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened bf8e1ef1f6635e037c32e6a8e41782e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10418712960, jitterRate=-0.029681742191314697}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:21:48,577 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for bf8e1ef1f6635e037c32e6a8e41782e2: 2023-11-12T09:21:48,578 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2., pid=28, masterSystemTime=1699780908555 2023-11-12T09:21:48,581 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:21:48,581 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:21:48,581 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=bf8e1ef1f6635e037c32e6a8e41782e2, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:48,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-11-12T09:21:48,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2, server=jenkins-hbase3.apache.org,44111,1699780893376 in 180 msec 2023-11-12T09:21:48,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-11-12T09:21:48,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, ASSIGN in 340 msec 2023-11-12T09:21:48,589 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:21:48,589 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780908589"}]},"ts":"1699780908589"} 2023-11-12T09:21:48,591 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-12T09:21:48,595 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:21:48,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 813 msec 2023-11-12T09:21:48,657 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:21:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-12T09:21:48,896 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-11-12T09:21:48,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$4(2333): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-12T09:21:48,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-11-12T09:21:48,916 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-11-12T09:21:48,916 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:48,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(712): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-11-12T09:21:48,917 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-12T09:21:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:21:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:21:49,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:21:49,330 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 334467069bbeef2313578e4dfe9024f3, NAME => 'backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe 2023-11-12T09:21:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:21:49,742 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:49,743 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 334467069bbeef2313578e4dfe9024f3, disabling compactions & flushes 2023-11-12T09:21:49,743 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. 2023-11-12T09:21:49,743 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. 2023-11-12T09:21:49,743 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. after waiting 0 ms 2023-11-12T09:21:49,743 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. 2023-11-12T09:21:49,743 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. 2023-11-12T09:21:49,743 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 334467069bbeef2313578e4dfe9024f3: 2023-11-12T09:21:49,744 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-11-12T09:21:49,744 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1699780909744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699780909744"}]},"ts":"1699780909744"} 2023-11-12T09:21:49,747 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:21:49,747 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-12T09:21:49,748 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780909748"}]},"ts":"1699780909748"} 2023-11-12T09:21:49,749 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-11-12T09:21:49,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=334467069bbeef2313578e4dfe9024f3, ASSIGN}] 2023-11-12T09:21:49,758 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=334467069bbeef2313578e4dfe9024f3, ASSIGN 2023-11-12T09:21:49,759 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=334467069bbeef2313578e4dfe9024f3, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,44111,1699780893376; forceNewPlan=false, retain=false 2023-11-12T09:21:49,910 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=334467069bbeef2313578e4dfe9024f3, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:49,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 334467069bbeef2313578e4dfe9024f3, server=jenkins-hbase3.apache.org,44111,1699780893376}] 2023-11-12T09:21:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:21:50,065 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:50,070 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. 2023-11-12T09:21:50,070 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 334467069bbeef2313578e4dfe9024f3, NAME => 'backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:21:50,071 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:21:50,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 334467069bbeef2313578e4dfe9024f3 2023-11-12T09:21:50,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:21:50,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 334467069bbeef2313578e4dfe9024f3 2023-11-12T09:21:50,072 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 334467069bbeef2313578e4dfe9024f3 2023-11-12T09:21:50,075 INFO [StoreOpener-334467069bbeef2313578e4dfe9024f3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 334467069bbeef2313578e4dfe9024f3 2023-11-12T09:21:50,077 INFO [StoreOpener-334467069bbeef2313578e4dfe9024f3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 334467069bbeef2313578e4dfe9024f3 columnFamilyName meta 2023-11-12T09:21:50,077 DEBUG [StoreOpener-334467069bbeef2313578e4dfe9024f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:50,078 INFO [StoreOpener-334467069bbeef2313578e4dfe9024f3-1 {}] regionserver.HStore(324): Store=334467069bbeef2313578e4dfe9024f3/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:21:50,078 INFO [StoreOpener-334467069bbeef2313578e4dfe9024f3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 334467069bbeef2313578e4dfe9024f3 2023-11-12T09:21:50,079 INFO [StoreOpener-334467069bbeef2313578e4dfe9024f3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 334467069bbeef2313578e4dfe9024f3 columnFamilyName session 2023-11-12T09:21:50,080 DEBUG [StoreOpener-334467069bbeef2313578e4dfe9024f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:21:50,080 INFO [StoreOpener-334467069bbeef2313578e4dfe9024f3-1 {}] regionserver.HStore(324): Store=334467069bbeef2313578e4dfe9024f3/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:21:50,081 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system_bulk/334467069bbeef2313578e4dfe9024f3 2023-11-12T09:21:50,082 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system_bulk/334467069bbeef2313578e4dfe9024f3 2023-11-12T09:21:50,084 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:21:50,087 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 334467069bbeef2313578e4dfe9024f3 2023-11-12T09:21:50,090 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system_bulk/334467069bbeef2313578e4dfe9024f3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-12T09:21:50,090 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 334467069bbeef2313578e4dfe9024f3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11788118720, jitterRate=0.09785410761833191}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:21:50,091 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 334467069bbeef2313578e4dfe9024f3: 2023-11-12T09:21:50,092 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3., pid=31, masterSystemTime=1699780910065 2023-11-12T09:21:50,094 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. 2023-11-12T09:21:50,094 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. 2023-11-12T09:21:50,095 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=334467069bbeef2313578e4dfe9024f3, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:50,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-11-12T09:21:50,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 334467069bbeef2313578e4dfe9024f3, server=jenkins-hbase3.apache.org,44111,1699780893376 in 184 msec 2023-11-12T09:21:50,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-11-12T09:21:50,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=334467069bbeef2313578e4dfe9024f3, ASSIGN in 343 msec 2023-11-12T09:21:50,103 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-12T09:21:50,103 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780910103"}]},"ts":"1699780910103"} 2023-11-12T09:21:50,104 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-11-12T09:21:50,108 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-11-12T09:21:50,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.1950 sec 2023-11-12T09:21:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-12T09:21:51,027 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-11-12T09:21:51,040 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:21:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44111 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2023-11-12T09:21:51,076 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:21:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699780911076 (current time:1699780911076). 2023-11-12T09:21:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:21:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:21:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x24ea9d79 to 127.0.0.1:56022 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:21:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fab35e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:21:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:21:51,090 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:33928, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:21:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x24ea9d79 to 127.0.0.1:56022 2023-11-12T09:21:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:21:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:21:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:21:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-12T09:21:51,108 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:21:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:21:51,114 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:21:51,130 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot, current working dir path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:21:51,133 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:21:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:21:51,340 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-12T09:21:51,340 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-11-12T09:21:51,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-11-12T09:21:51,341 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-11-12T09:21:51,342 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16997809018503 2023-11-12T09:21:51,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16997809018503 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:21:51,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1699780901850 2023-11-12T09:21:51,343 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1699780901850 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:21:51,344 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997809018501 2023-11-12T09:21:51,344 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16997809018501 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:21:51,345 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16997809018502 2023-11-12T09:21:51,345 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16997809018502 Metrics about Tables on a single HBase RegionServer 2023-11-12T09:21:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:21:51,544 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:21:51,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2}] 2023-11-12T09:21:51,555 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:21:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:21:51,715 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:51,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44111 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-11-12T09:21:51,718 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:21:51,721 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing bf8e1ef1f6635e037c32e6a8e41782e2 2/2 column families, dataSize=45 B heapSize=632 B 2023-11-12T09:21:51,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/0385264caa2440b4b46b20a5b930cc8c is 49, key is activesession:/session:c/1699780911062/Put/seqid=0 2023-11-12T09:21:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:21:52,263 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/0385264caa2440b4b46b20a5b930cc8c 2023-11-12T09:21:52,338 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/0385264caa2440b4b46b20a5b930cc8c as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c 2023-11-12T09:21:52,346 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c, entries=1, sequenceid=5, filesize=4.9 K 2023-11-12T09:21:52,350 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for bf8e1ef1f6635e037c32e6a8e41782e2 in 629ms, sequenceid=5, compaction requested=false 2023-11-12T09:21:52,350 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-12T09:21:52,351 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for bf8e1ef1f6635e037c32e6a8e41782e2: 2023-11-12T09:21:52,352 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. for snapshot_backup_system completed. 2023-11-12T09:21:52,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:21:52,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:21:52,363 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-12T09:21:52,364 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c] hfiles 2023-11-12T09:21:52,364 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c for snapshot=snapshot_backup_system 2023-11-12T09:21:52,783 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:21:52,788 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-11-12T09:21:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster(4026): Remote procedure done, pid=33 2023-11-12T09:21:52,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:21:52,795 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:21:52,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-11-12T09:21:52,799 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:21:52,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 in 1.2490 sec 2023-11-12T09:21:52,802 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:21:52,806 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:21:52,807 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:21:52,810 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:21:52,845 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:21:52,862 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:21:52,862 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:21:52,865 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:21:52,865 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-12T09:21:52,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.7640 sec 2023-11-12T09:21:53,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-12T09:21:53,218 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-11-12T09:21:53,218 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(115): Backup backup_1699780907622 started at 1699780913218. 2023-11-12T09:21:53,242 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(119): Backup session backup_1699780907622 has been started. 2023-11-12T09:21:53,246 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:21:53,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:21:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:21:53,264 INFO [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:21:53,264 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:21:53,265 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:21:53,265 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:21:53,265 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:21:53,265 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:21:53,267 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:21:53,267 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,267 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:21:53,267 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:21:53,268 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,268 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:21:53,268 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:21:53,268 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-12T09:21:53,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:21:53,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:21:53,269 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:21:53,273 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:21:53,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:21:53,273 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:21:53,273 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:21:53,274 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:21:53,274 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:21:53,274 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44111,1699780893376' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:21:53,283 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,283 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:21:53,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:21:53,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:21:53,283 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-12T09:21:53,283 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:21:53,283 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:21:53,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:21:53,284 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:21:53,285 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,285 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:21:53,286 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44111,1699780893376' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:21:53,286 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1aa20edb[Count = 0] remaining members to acquire global barrier 2023-11-12T09:21:53,286 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:21:53,286 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:21:53,288 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:21:53,288 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:21:53,289 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:21:53,289 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:21:53,289 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,289 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:21:53,290 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:21:53,290 DEBUG [rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,290 INFO [rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699780895119 highest: 1699780895119 on jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,291 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta:.meta(num 1699780895737) roll requested 2023-11-12T09:21:53,307 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780913291.meta, exclude list is [], retry=0 2023-11-12T09:21:53,316 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK] 2023-11-12T09:21:53,319 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780895737.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780913291.meta 2023-11-12T09:21:53,320 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK]] 2023-11-12T09:21:53,320 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780895737.meta is not closed yet, will try archiving it next time 2023-11-12T09:21:53,320 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44111%2C1699780893376:(num 1699780895119) roll requested 2023-11-12T09:21:53,332 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780913320, exclude list is [], retry=0 2023-11-12T09:21:53,336 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK] 2023-11-12T09:21:53,340 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780895119 with entries=19, filesize=3.99 KB; new WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780913320 2023-11-12T09:21:53,340 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK]] 2023-11-12T09:21:53,340 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780895119 is not closed yet, will try archiving it next time 2023-11-12T09:21:53,344 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780895119 not finished, retry = 0 2023-11-12T09:21:53,352 DEBUG [rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 62 2023-11-12T09:21:53,352 INFO [rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699780913320 on jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,352 DEBUG [rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:21:53,354 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:32880, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-12T09:21:53,375 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:21:53,375 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44111,1699780893376' in zk 2023-11-12T09:21:53,378 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:21:53,378 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,378 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:21:53,378 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:21:53,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:21:53,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:21:53,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:21:53,381 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:21:53,381 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:21:53,381 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,382 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:21:53,382 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:21:53,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44111,1699780893376': 2023-11-12T09:21:53,384 INFO [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:21:53,384 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44111,1699780893376' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:21:53,384 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:21:53,384 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:21:53,384 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:21:53,384 INFO [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:21:53,388 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:21:53,388 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:21:53,388 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:21:53,388 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:21:53,388 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:21:53,388 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:21:53,388 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:21:53,388 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:21:53,388 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:21:53,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:21:53,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:21:53,389 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:21:53,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:21:53,390 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:21:53,390 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:21:53,391 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,392 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,392 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:21:53,392 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:21:53,393 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,406 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,406 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:21:53,406 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-12T09:21:53,406 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-12T09:21:53,406 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44111-0x1004ceff8aa0001, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-12T09:21:53,406 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:21:53,406 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,406 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:21:53,406 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:21:53,406 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-12T09:21:53,406 DEBUG [Listener at localhost/45829-EventThread {}] zookeeper.ZKWatcher(604): master:33463-0x1004ceff8aa0000, quorum=127.0.0.1:56022, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-12T09:21:53,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:21:53,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:21:53,407 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:21:53,407 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:21:53,407 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:21:53,410 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 100000 ms per retry) 2023-11-12T09:21:53,410 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-12T09:21:53,413 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-11-12T09:21:53,510 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-12T09:21:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:21:53,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } 2023-11-12T09:21:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699780913524 (current time:1699780913524). 2023-11-12T09:21:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:21:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699780913523_default_test-1699780901850 VERSION not specified, setting to 2 2023-11-12T09:21:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7714267e to 127.0.0.1:56022 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:21:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@265fd34d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:21:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:21:53,540 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:33936, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:21:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7714267e to 127.0.0.1:56022 2023-11-12T09:21:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:21:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:21:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } 2023-11-12T09:21:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-11-12T09:21:53,546 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:21:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:21:53,547 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:21:53,551 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot, current working dir path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_1699780913523_default_test-1699780901850 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:21:53,553 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:21:53,570 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:21:53,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35}] 2023-11-12T09:21:53,572 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:21:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:21:53,723 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:53,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44111 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-11-12T09:21:53,725 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:21:53,727 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 9d3fa1e74299c566cf2ed2c7ca01ec35 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-12T09:21:53,755 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/.tmp/f/366cfc2b3047493493ca939e4ba045cc is 37, key is row10/f:q1/1699780903502/Put/seqid=0 2023-11-12T09:21:53,771 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/.tmp/f/366cfc2b3047493493ca939e4ba045cc 2023-11-12T09:21:53,782 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/.tmp/f/366cfc2b3047493493ca939e4ba045cc as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc 2023-11-12T09:21:53,802 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc, entries=99, sequenceid=103, filesize=8.2 K 2023-11-12T09:21:53,804 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 9d3fa1e74299c566cf2ed2c7ca01ec35 in 77ms, sequenceid=103, compaction requested=false 2023-11-12T09:21:53,805 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 9d3fa1e74299c566cf2ed2c7ca01ec35: 2023-11-12T09:21:53,806 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. for snapshot_1699780913523_default_test-1699780901850 completed. 2023-11-12T09:21:53,806 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35.' region-info for snapshot=snapshot_1699780913523_default_test-1699780901850 2023-11-12T09:21:53,806 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:21:53,806 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc] hfiles 2023-11-12T09:21:53,806 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc for snapshot=snapshot_1699780913523_default_test-1699780901850 2023-11-12T09:21:53,820 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:21:53,820 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-11-12T09:21:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster(4026): Remote procedure done, pid=35 2023-11-12T09:21:53,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699780913523_default_test-1699780901850 on region 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:21:53,822 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:21:53,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-11-12T09:21:53,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35 in 253 msec 2023-11-12T09:21:53,826 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:21:53,828 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:21:53,828 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:21:53,828 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699780913523_default_test-1699780901850 2023-11-12T09:21:53,829 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_1699780913523_default_test-1699780901850 2023-11-12T09:21:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:21:54,069 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:21:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:21:54,246 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:21:54,254 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:21:54,255 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_1699780913523_default_test-1699780901850 to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_1699780913523_default_test-1699780901850 2023-11-12T09:21:54,256 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:21:54,256 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-11-12T09:21:54,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1699780913523_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } in 714 msec 2023-11-12T09:21:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-12T09:21:54,654 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: default:test-1699780901850, procId: 34 completed 2023-11-12T09:21:54,654 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699780907622 2023-11-12T09:21:54,654 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:21:54,656 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-12T09:21:54,658 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699780907622_test-1699780901850 2023-11-12T09:21:54,658 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699780913523_default_test-1699780901850 to hdfs://localhost:37959/backupUT/backup_1699780907622/default/test-1699780901850/ 2023-11-12T09:21:54,668 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:21:54,699 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37959, inputRoot=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe 2023-11-12T09:21:54,700 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33463,1699780890205_-2117378274_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37959/backupUT/backup_1699780907622/default/test-1699780901850, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37959/backupUT/backup_1699780907622/default/test-1699780901850/.hbase-snapshot/.tmp/snapshot_1699780913523_default_test-1699780901850 2023-11-12T09:21:54,700 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37959, inputRoot=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe, snapshotDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_1699780913523_default_test-1699780901850. 2023-11-12T09:21:54,712 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_1699780913523_default_test-1699780901850 to hdfs://localhost:37959/backupUT/backup_1699780907622/default/test-1699780901850/.hbase-snapshot/.tmp/snapshot_1699780913523_default_test-1699780901850 2023-11-12T09:21:55,149 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:21:55,150 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:21:55,150 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:21:55,151 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:21:56,147 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/hadoop-355514630910847824.jar 2023-11-12T09:21:56,147 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:21:56,148 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:21:56,148 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:21:56,149 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:21:56,149 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:21:56,150 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:21:56,150 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:21:56,150 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:21:56,151 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:21:56,152 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:21:56,152 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:21:56,153 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:21:56,153 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:21:56,154 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:21:56,154 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:21:56,155 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:21:56,155 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:21:56,156 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:21:56,156 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:21:56,157 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:21:56,157 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:21:56,161 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:21:56,162 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:21:56,162 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:21:56,163 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:21:56,163 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:21:56,164 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:21:56,164 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:21:56,229 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-12T09:21:56,252 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699780913523_default_test-1699780901850' hfile list 2023-11-12T09:21:56,263 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-12T09:21:56,520 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/mapreduce.cluster.local.dir/1699780916471/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:21:56,521 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/mapreduce.cluster.local.dir/1699780916471/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:21:56,721 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:21:56,751 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc output=hdfs://localhost:37959/backupUT/backup_1699780907622/default/test-1699780901850/archive/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc 2023-11-12T09:21:56,751 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-11-12T09:21:57,603 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:21:57,604 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:21:57,612 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699780913523_default_test-1699780901850 2023-11-12T09:21:57,612 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699780913523_default_test-1699780901850 finished. 2023-11-12T09:21:57,613 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1034): test-1699780901850 2023-11-12T09:21:57,694 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699780907622 2023-11-12T09:21:57,694 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:21:58,113 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37959/backupUT/backup_1699780907622/.backup.manifest 2023-11-12T09:21:58,113 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(358): Backup backup_1699780907622 finished: type=FULL,tablelist=test-1699780901850,targetRootDir=hdfs://localhost:37959/backupUT,startts=1699780913218,completets=1699780917691,bytescopied=0 2023-11-12T09:21:58,113 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:21:58,113 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699780913523_default_test-1699780901850 2023-11-12T09:21:58,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699780913523_default_test-1699780901850" 2023-11-12T09:21:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699780913523_default_test-1699780901850 2023-11-12T09:21:58,126 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699780913523_default_test-1699780901850 for backup backup_1699780907622 succeeded. 2023-11-12T09:21:58,127 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:21:58,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:21:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:21:58,142 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:21:58,150 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:21:58,168 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(376): Backup backup_1699780907622 completed. 2023-11-12T09:21:58,168 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-12T09:21:58,168 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3550ce00 to 127.0.0.1:56022 2023-11-12T09:21:58,168 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:21:58,188 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] backup.TestBackupDeleteWithFailures(147): backup complete 2023-11-12T09:21:58,213 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:21:58,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:21:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699780918222 (current time:1699780918222). 2023-11-12T09:21:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:21:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:21:58,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0a8ba18e to 127.0.0.1:56022 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:21:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3951b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:21:58,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:21:58,231 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36802, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:21:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0a8ba18e to 127.0.0.1:56022 2023-11-12T09:21:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:21:58,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:21:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:21:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-11-12T09:21:58,235 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:21:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:21:58,237 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:21:58,240 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot, current working dir path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:21:58,243 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:21:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:21:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:21:58,651 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:21:58,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2}] 2023-11-12T09:21:58,654 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:21:58,805 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:21:58,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44111 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-11-12T09:21:58,806 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:21:58,806 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2819): Flushing bf8e1ef1f6635e037c32e6a8e41782e2 2/2 column families, dataSize=1.14 KB heapSize=2.37 KB 2023-11-12T09:21:58,823 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/442ef22cf4f0499c91e911cb5d9a70f0 is 169, key is trslm:hdfs://localhost:37959/backupUT\x00test-1699780901850/meta:log-roll-map/1699780917681/Put/seqid=0 2023-11-12T09:21:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:21:59,233 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=586 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/442ef22cf4f0499c91e911cb5d9a70f0 2023-11-12T09:21:59,244 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/efa79550ed004f0c9565636ed190b0d0 is 310, key is session:backup_1699780907622/session:context/1699780918148/Put/seqid=0 2023-11-12T09:21:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:21:59,655 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/efa79550ed004f0c9565636ed190b0d0 2023-11-12T09:21:59,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/442ef22cf4f0499c91e911cb5d9a70f0 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0 2023-11-12T09:21:59,672 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0, entries=5, sequenceid=18, filesize=5.5 K 2023-11-12T09:21:59,674 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/efa79550ed004f0c9565636ed190b0d0 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0 2023-11-12T09:21:59,682 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0, entries=2, sequenceid=18, filesize=5.3 K 2023-11-12T09:21:59,683 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1165, heapSize ~2.34 KB/2392, currentSize=0 B/0 for bf8e1ef1f6635e037c32e6a8e41782e2 in 877ms, sequenceid=18, compaction requested=false 2023-11-12T09:21:59,683 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for bf8e1ef1f6635e037c32e6a8e41782e2: 2023-11-12T09:21:59,683 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. for snapshot_backup_system completed. 2023-11-12T09:21:59,683 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:21:59,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:21:59,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0] hfiles 2023-11-12T09:21:59,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0 for snapshot=snapshot_backup_system 2023-11-12T09:21:59,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0] hfiles 2023-11-12T09:21:59,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c for snapshot=snapshot_backup_system 2023-11-12T09:21:59,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0 for snapshot=snapshot_backup_system 2023-11-12T09:22:00,097 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:00,097 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-11-12T09:22:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster(4026): Remote procedure done, pid=37 2023-11-12T09:22:00,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:00,099 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:00,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-11-12T09:22:00,102 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:22:00,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 in 1.4480 sec 2023-11-12T09:22:00,103 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:22:00,104 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:22:00,104 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:22:00,105 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:22:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:22:00,522 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:22:00,531 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:22:00,532 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:22:00,533 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:22:00,533 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-11-12T09:22:00,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.3010 sec 2023-11-12T09:22:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-12T09:22:02,346 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 36 completed 2023-11-12T09:22:02,354 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(227): Deleting backup backup_1699780907622 ... 2023-11-12T09:22:02,355 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(437): Delete log files: .backup.manifest 2023-11-12T09:22:02,356 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(437): Delete log files: default 2023-11-12T09:22:02,357 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(445): Trying to cleanup up target dir : backup_1699780907622 2023-11-12T09:22:02,358 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(460): No data has been found in hdfs://localhost:37959/backupUT/backup_1699780907622/default/test-1699780901850. 2023-11-12T09:22:02,358 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(467): hdfs://localhost:37959/backupUT/backup_1699780907622/default is empty, remove it. 2023-11-12T09:22:02,367 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-11-12T09:22:02,367 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-11-12T09:22:02,371 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(276): Delete backup backup_1699780907622 completed. 2023-11-12T09:22:02,378 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1034): test-1699780901850 2023-11-12T09:22:02,381 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:22:02,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:22:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1209) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-12T09:22:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.CallRunner(138): callId: 143 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:52850 deadline: 1699780982387, exception=java.io.IOException: preDeleteSnapshot 2023-11-12T09:22:02,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:22:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1209) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-12T09:22:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.CallRunner(138): callId: 145 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:52850 deadline: 1699780982493, exception=java.io.IOException: preDeleteSnapshot 2023-11-12T09:22:02,503 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-12T09:22:02,509 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$18(967): Started disable of backup:system 2023-11-12T09:22:02,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable backup:system 2023-11-12T09:22:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-12T09:22:02,527 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780922527"}]},"ts":"1699780922527"} 2023-11-12T09:22:02,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:22:02,528 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-12T09:22:02,530 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-12T09:22:02,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, UNASSIGN}] 2023-11-12T09:22:02,534 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, UNASSIGN 2023-11-12T09:22:02,534 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=bf8e1ef1f6635e037c32e6a8e41782e2, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:02,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; CloseRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2, server=jenkins-hbase3.apache.org,44111,1699780893376}] 2023-11-12T09:22:02,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:22:02,695 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:02,697 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(119): Close bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:02,697 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1668): Closing bf8e1ef1f6635e037c32e6a8e41782e2, disabling compactions & flushes 2023-11-12T09:22:02,697 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1690): Closing region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:02,697 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:02,697 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. after waiting 0 ms 2023-11-12T09:22:02,697 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:02,697 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(2819): Flushing bf8e1ef1f6635e037c32e6a8e41782e2 2/2 column families, dataSize=179 B heapSize=912 B 2023-11-12T09:22:02,703 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/e74f8da1eb58437d82cbb2bc2592ed2b is 91, key is incrbackupset:hdfs://localhost:37959/backupUT/meta:test-1699780901850/1699780922378/Put/seqid=0 2023-11-12T09:22:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:22:03,109 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/e74f8da1eb58437d82cbb2bc2592ed2b 2023-11-12T09:22:03,116 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e74f8da1eb58437d82cbb2bc2592ed2b 2023-11-12T09:22:03,120 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/f09a8671a4294fdebdb4635781cbbb36 is 59, key is session:backup_1699780907622/session:/1699780922370/DeleteFamily/seqid=0 2023-11-12T09:22:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:22:03,525 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55 B at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/f09a8671a4294fdebdb4635781cbbb36 2023-11-12T09:22:03,532 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f09a8671a4294fdebdb4635781cbbb36 2023-11-12T09:22:03,533 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/e74f8da1eb58437d82cbb2bc2592ed2b as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/e74f8da1eb58437d82cbb2bc2592ed2b 2023-11-12T09:22:03,539 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e74f8da1eb58437d82cbb2bc2592ed2b 2023-11-12T09:22:03,539 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/e74f8da1eb58437d82cbb2bc2592ed2b, entries=2, sequenceid=24, filesize=5.2 K 2023-11-12T09:22:03,541 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/f09a8671a4294fdebdb4635781cbbb36 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/f09a8671a4294fdebdb4635781cbbb36 2023-11-12T09:22:03,547 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f09a8671a4294fdebdb4635781cbbb36 2023-11-12T09:22:03,547 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/f09a8671a4294fdebdb4635781cbbb36, entries=1, sequenceid=24, filesize=5.1 K 2023-11-12T09:22:03,548 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(3022): Finished flush of dataSize ~179 B/179, heapSize ~880 B/880, currentSize=0 B/0 for bf8e1ef1f6635e037c32e6a8e41782e2 in 851ms, sequenceid=24, compaction requested=true 2023-11-12T09:22:03,561 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1 2023-11-12T09:22:03,561 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:22:03,562 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1909): Closed backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:03,562 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1622): Region close journal for bf8e1ef1f6635e037c32e6a8e41782e2: 2023-11-12T09:22:03,565 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(163): Closed bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:03,565 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=bf8e1ef1f6635e037c32e6a8e41782e2, regionState=CLOSED 2023-11-12T09:22:03,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-11-12T09:22:03,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; CloseRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2, server=jenkins-hbase3.apache.org,44111,1699780893376 in 1.0310 sec 2023-11-12T09:22:03,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=39, resume processing ppid=38 2023-11-12T09:22:03,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, ppid=38, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, UNASSIGN in 1.0370 sec 2023-11-12T09:22:03,572 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780923572"}]},"ts":"1699780923572"} 2023-11-12T09:22:03,574 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-12T09:22:03,576 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-12T09:22:03,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0610 sec 2023-11-12T09:22:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-12T09:22:03,634 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$TableFuture(3745): Operation: DISABLE, Table Name: backup:system, procId: 38 completed 2023-11-12T09:22:03,644 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin(2800): Taking restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1699780923644 2023-11-12T09:22:03,645 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699780923645 (current time:1699780923645). 2023-11-12T09:22:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:22:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1699780923644 VERSION not specified, setting to 2 2023-11-12T09:22:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0d33a617 to 127.0.0.1:56022 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-12T09:22:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4921127e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:03,655 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36804, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:22:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0d33a617 to 127.0.0.1:56022 2023-11-12T09:22:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:22:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=41, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-11-12T09:22:03,661 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:22:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:22:03,662 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:22:03,664 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot, current working dir path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699780923644 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:22:03,668 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:22:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:22:03,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:22:04,075 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-12T09:22:04,077 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-12T09:22:04,077 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-12T09:22:04,077 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:22:04,079 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/e74f8da1eb58437d82cbb2bc2592ed2b] hfiles 2023-11-12T09:22:04,079 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0 2023-11-12T09:22:04,079 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/e74f8da1eb58437d82cbb2bc2592ed2b 2023-11-12T09:22:04,079 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:22:04,081 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/f09a8671a4294fdebdb4635781cbbb36] hfiles 2023-11-12T09:22:04,081 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c 2023-11-12T09:22:04,081 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0 2023-11-12T09:22:04,081 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/f09a8671a4294fdebdb4635781cbbb36 2023-11-12T09:22:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:22:04,488 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => bf8e1ef1f6635e037c32e6a8e41782e2, NAME => 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-12T09:22:04,489 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:22:04,489 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:22:04,489 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1699780923644 2023-11-12T09:22:04,490 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699780923644 2023-11-12T09:22:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:22:04,904 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:22:04,913 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:22:04,914 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699780923644 to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1699780923644 2023-11-12T09:22:04,915 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:22:04,915 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-11-12T09:22:04,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780923644 table=backup:system type=FLUSH ttl=0 } in 1.2580 sec 2023-11-12T09:22:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-12T09:22:05,768 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 41 completed 2023-11-12T09:22:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699780918222 type: FLUSH version: 2 ttl: 0 ) 2023-11-12T09:22:05,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-12T09:22:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-12T09:22:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-12T09:22:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-12T09:22:06,220 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-11-12T09:22:06,220 INFO [PEWorker-2 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-11-12T09:22:06,222 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:06,228 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699780918222 type: FLUSH version: 2 ttl: 0 2023-11-12T09:22:06,228 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system 2023-11-12T09:22:06,232 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-12T09:22:06,232 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(227): region to restore: bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:06,246 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/e74f8da1eb58437d82cbb2bc2592ed2b to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/archive/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/e74f8da1eb58437d82cbb2bc2592ed2b 2023-11-12T09:22:06,247 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:22:06,251 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/f09a8671a4294fdebdb4635781cbbb36 to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/archive/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/f09a8671a4294fdebdb4635781cbbb36 2023-11-12T09:22:06,251 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:22:06,251 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699780918222 type: FLUSH version: 2 ttl: 0 2023-11-12T09:22:06,252 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699780926252"}]},"ts":"9223372036854775807"} 2023-11-12T09:22:06,255 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:22:06,255 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => bf8e1ef1f6635e037c32e6a8e41782e2, NAME => 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:22:06,256 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1699780926253"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699780926253"}]},"ts":"1699780926253"} 2023-11-12T09:22:06,257 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:22:06,257 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-12T09:22:06,257 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => bf8e1ef1f6635e037c32e6a8e41782e2, NAME => 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:22:06,258 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-12T09:22:06,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699780918222 type: FLUSH version: 2 ttl: 0 ) in 470 msec 2023-11-12T09:22:06,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2023-11-12T09:22:06,412 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$TableFuture(3745): Operation: MODIFY, Table Name: backup:system, procId: 42 completed 2023-11-12T09:22:06,412 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin(2833): Deleting restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1699780923644 2023-11-12T09:22:06,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1699780923644" 2023-11-12T09:22:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1209) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-12T09:22:06,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.CallRunner(138): callId: 192 service: MasterService methodName: DeleteSnapshot size: 86 connection: 172.31.12.81:52850 deadline: 1699780986413, exception=java.io.IOException: preDeleteSnapshot 2023-11-12T09:22:06,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1699780923644" 2023-11-12T09:22:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1209) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-12T09:22:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.CallRunner(138): callId: 194 service: MasterService methodName: DeleteSnapshot size: 86 connection: 172.31.12.81:52850 deadline: 1699780986518, exception=java.io.IOException: preDeleteSnapshot 2023-11-12T09:22:06,522 ERROR [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin(2836): Unable to remove the failsafe snapshot: hbase-failsafe-snapshot_backup_system-1699780923644 org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2023-11-12T09:22:06.417Z, RpcRetryingCaller{globalStartTime=2023-11-12T09:22:06.412Z, pause=100, maxAttempts=2}, java.io.IOException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1209) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1206) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1206) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2023-11-12T09:22:06.522Z, RpcRetryingCaller{globalStartTime=2023-11-12T09:22:06.412Z, pause=100, maxAttempts=2}, java.io.IOException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1209) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1206) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1206) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3173) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3165) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.deleteSnapshot(HBaseAdmin.java:3073) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.restoreSnapshot(HBaseAdmin.java:2834) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.restoreSnapshot(HBaseAdmin.java:2772) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.restoreSnapshot(HBaseAdmin.java:2737) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.backup.impl.BackupSystemTable.restoreFromSnapshot(BackupSystemTable.java:1543) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.deleteBackups(BackupAdminImpl.java:155) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailuresAfter(TestBackupDeleteWithFailures.java:161) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailures(TestBackupDeleteWithFailures.java:136) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1209) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1206) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1206) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.MasterCallable.call(MasterCallable.java:101) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 34 more Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1209) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1206) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1206) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more 2023-11-12T09:22:06,526 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$17(907): Started enable of backup:system 2023-11-12T09:22:06,531 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$12(2662): Client=jenkins//172.31.12.81 enable backup:system 2023-11-12T09:22:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=43, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-12T09:22:06,543 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-12T09:22:06,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:22:06,543 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780926543"}]},"ts":"1699780926543"} 2023-11-12T09:22:06,545 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-12T09:22:06,549 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-12T09:22:06,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, ASSIGN}] 2023-11-12T09:22:06,552 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, ASSIGN 2023-11-12T09:22:06,552 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-12T09:22:06,601 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-12T09:22:06,604 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:41248, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-12T09:22:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:22:06,704 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=44 updating hbase:meta row=bf8e1ef1f6635e037c32e6a8e41782e2, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:06,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE; OpenRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2, server=jenkins-hbase3.apache.org,44111,1699780893376}] 2023-11-12T09:22:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:22:06,859 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:06,863 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(130): Open backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:06,863 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7254): Opening region: {ENCODED => bf8e1ef1f6635e037c32e6a8e41782e2, NAME => 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:22:06,864 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:22:06,864 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:06,864 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(888): Instantiated backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:22:06,864 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7296): checking encryption for bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:06,864 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7299): checking classloading for bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:06,866 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:06,866 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf8e1ef1f6635e037c32e6a8e41782e2 columnFamilyName meta 2023-11-12T09:22:06,867 DEBUG [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:22:06,873 DEBUG [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0 2023-11-12T09:22:06,874 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.HStore(324): Store=bf8e1ef1f6635e037c32e6a8e41782e2/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:22:06,874 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:06,875 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf8e1ef1f6635e037c32e6a8e41782e2 columnFamilyName session 2023-11-12T09:22:06,875 DEBUG [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:22:06,881 DEBUG [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c 2023-11-12T09:22:06,887 DEBUG [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0 2023-11-12T09:22:06,887 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.HStore(324): Store=bf8e1ef1f6635e037c32e6a8e41782e2/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:22:06,888 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:06,889 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:06,891 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:22:06,892 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1076): writing seq id for bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:06,893 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): Opened bf8e1ef1f6635e037c32e6a8e41782e2; next sequenceid=28; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9586923360, jitterRate=-0.1071481853723526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:22:06,894 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(992): Region open journal for bf8e1ef1f6635e037c32e6a8e41782e2: 2023-11-12T09:22:06,895 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2., pid=45, masterSystemTime=1699780926859 2023-11-12T09:22:06,897 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:06,897 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(158): Opened backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:06,898 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=44 updating hbase:meta row=bf8e1ef1f6635e037c32e6a8e41782e2, regionState=OPEN, openSeqNum=28, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:06,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-11-12T09:22:06,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; OpenRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2, server=jenkins-hbase3.apache.org,44111,1699780893376 in 193 msec 2023-11-12T09:22:06,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=44, resume processing ppid=43 2023-11-12T09:22:06,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, ppid=43, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, ASSIGN in 353 msec 2023-11-12T09:22:06,905 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780926904"}]},"ts":"1699780926904"} 2023-11-12T09:22:06,906 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-12T09:22:06,908 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-12T09:22:06,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, state=SUCCESS; EnableTableProcedure table=backup:system in 376 msec 2023-11-12T09:22:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-12T09:22:07,148 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] client.HBaseAdmin$TableFuture(3745): Operation: ENABLE, Table Name: backup:system, procId: 43 completed 2023-11-12T09:22:07,148 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-12T09:22:07,148 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:22:07,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:22:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1209) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-12T09:22:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.CallRunner(138): callId: 208 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:52850 deadline: 1699780987156, exception=java.io.IOException: preDeleteSnapshot 2023-11-12T09:22:07,262 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:22:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1209) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$92.call(MasterCoprocessorHost.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-12T09:22:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.CallRunner(138): callId: 210 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:52850 deadline: 1699780987262, exception=java.io.IOException: preDeleteSnapshot 2023-11-12T09:22:07,266 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:22:07,294 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251f9436, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:07,302 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3c6798, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:07,303 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:22:07,304 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:41250, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:22:07,308 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:07,309 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45776, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService REPAIR status: no failed sessions found. Checking failed delete backup operation ... Found failed DELETE operation for: backup_1699780907622 Running DELETE again ... 2023-11-12T09:22:07,324 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-12T09:22:07,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$13(2695): Client=jenkins//172.31.12.81 disable backup:system 2023-11-12T09:22:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=46, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-12T09:22:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2023-11-12T09:22:07,337 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780927336"}]},"ts":"1699780927336"} 2023-11-12T09:22:07,338 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-12T09:22:07,340 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-12T09:22:07,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, UNASSIGN}] 2023-11-12T09:22:07,342 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, UNASSIGN 2023-11-12T09:22:07,342 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=47 updating hbase:meta row=bf8e1ef1f6635e037c32e6a8e41782e2, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:07,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; CloseRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2, server=jenkins-hbase3.apache.org,44111,1699780893376}] 2023-11-12T09:22:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2023-11-12T09:22:07,495 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:07,496 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] handler.UnassignRegionHandler(119): Close bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:07,497 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1668): Closing bf8e1ef1f6635e037c32e6a8e41782e2, disabling compactions & flushes 2023-11-12T09:22:07,497 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1690): Closing region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:07,497 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:07,497 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. after waiting 0 ms 2023-11-12T09:22:07,497 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:07,497 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(2819): Flushing bf8e1ef1f6635e037c32e6a8e41782e2 2/2 column families, dataSize=44 B heapSize=632 B 2023-11-12T09:22:07,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/55bf9d8dfd8645659fd12358b1124e20 is 48, key is activesession:/session:c/1699780927267/Put/seqid=0 2023-11-12T09:22:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2023-11-12T09:22:07,917 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/55bf9d8dfd8645659fd12358b1124e20 2023-11-12T09:22:07,924 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/55bf9d8dfd8645659fd12358b1124e20 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/55bf9d8dfd8645659fd12358b1124e20 2023-11-12T09:22:07,931 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/55bf9d8dfd8645659fd12358b1124e20, entries=1, sequenceid=31, filesize=4.9 K 2023-11-12T09:22:07,932 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(3022): Finished flush of dataSize ~44 B/44, heapSize ~360 B/360, currentSize=0 B/0 for bf8e1ef1f6635e037c32e6a8e41782e2 in 435ms, sequenceid=31, compaction requested=true 2023-11-12T09:22:07,933 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-12T09:22:07,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2023-11-12T09:22:07,946 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/recovered.edits/34.seqid, newMaxSeqId=34, maxSeqId=27 2023-11-12T09:22:07,947 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:22:07,947 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1909): Closed backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:07,947 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1622): Region close journal for bf8e1ef1f6635e037c32e6a8e41782e2: 2023-11-12T09:22:07,950 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] handler.UnassignRegionHandler(163): Closed bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:07,951 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=47 updating hbase:meta row=bf8e1ef1f6635e037c32e6a8e41782e2, regionState=CLOSED 2023-11-12T09:22:07,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=48, resume processing ppid=47 2023-11-12T09:22:07,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, ppid=47, state=SUCCESS; CloseRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2, server=jenkins-hbase3.apache.org,44111,1699780893376 in 609 msec 2023-11-12T09:22:07,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=47, resume processing ppid=46 2023-11-12T09:22:07,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, ppid=46, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, UNASSIGN in 614 msec 2023-11-12T09:22:07,957 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780927957"}]},"ts":"1699780927957"} 2023-11-12T09:22:07,958 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-12T09:22:07,961 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-12T09:22:07,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, state=SUCCESS; DisableTableProcedure table=backup:system in 629 msec 2023-11-12T09:22:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2023-11-12T09:22:08,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699780928457 (current time:1699780928457). 2023-11-12T09:22:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:22:08,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1699780928456 VERSION not specified, setting to 2 2023-11-12T09:22:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32947e30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:08,466 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45788, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:22:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:08,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:22:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=49, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 49 2023-11-12T09:22:08,471 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:22:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-12T09:22:08,472 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:22:08,474 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot, current working dir path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699780928456 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:22:08,478 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:22:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-12T09:22:08,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-12T09:22:08,885 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-12T09:22:08,886 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-12T09:22:08,886 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-12T09:22:08,886 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:22:08,887 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0] hfiles 2023-11-12T09:22:08,887 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0 2023-11-12T09:22:08,887 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:22:08,889 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/55bf9d8dfd8645659fd12358b1124e20, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0] hfiles 2023-11-12T09:22:08,889 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c 2023-11-12T09:22:08,889 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/55bf9d8dfd8645659fd12358b1124e20 2023-11-12T09:22:08,889 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0 2023-11-12T09:22:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-12T09:22:09,110 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:22:09,295 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => bf8e1ef1f6635e037c32e6a8e41782e2, NAME => 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-12T09:22:09,297 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:22:09,298 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:22:09,298 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1699780928456 2023-11-12T09:22:09,299 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699780928456 2023-11-12T09:22:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-12T09:22:09,710 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:22:09,719 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:22:09,719 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1699780928456 to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1699780928456 2023-11-12T09:22:09,721 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:22:09,721 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 49 2023-11-12T09:22:09,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1699780928456 table=backup:system type=FLUSH ttl=0 } in 1.2530 sec 2023-11-12T09:22:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-12T09:22:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699780918222 type: FLUSH version: 2 ttl: 0 ) 2023-11-12T09:22:10,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-12T09:22:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:22:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:22:10,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:22:11,002 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-11-12T09:22:11,002 INFO [PEWorker-2 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-11-12T09:22:11,004 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:11,007 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699780918222 type: FLUSH version: 2 ttl: 0 2023-11-12T09:22:11,007 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system 2023-11-12T09:22:11,010 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-12T09:22:11,010 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(227): region to restore: bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:11,014 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:22:11,018 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/55bf9d8dfd8645659fd12358b1124e20 to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/archive/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/55bf9d8dfd8645659fd12358b1124e20 2023-11-12T09:22:11,018 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:22:11,019 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699780918222 type: FLUSH version: 2 ttl: 0 2023-11-12T09:22:11,020 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1699780931020"}]},"ts":"9223372036854775807"} 2023-11-12T09:22:11,022 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-12T09:22:11,022 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => bf8e1ef1f6635e037c32e6a8e41782e2, NAME => 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:22:11,023 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1699780931021"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1699780931021"}]},"ts":"1699780931021"} 2023-11-12T09:22:11,024 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-12T09:22:11,024 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-12T09:22:11,024 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => bf8e1ef1f6635e037c32e6a8e41782e2, NAME => 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.', STARTKEY => '', ENDKEY => ''}] 2023-11-12T09:22:11,024 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-12T09:22:11,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1699780918222 type: FLUSH version: 2 ttl: 0 ) in 438 msec 2023-11-12T09:22:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-12T09:22:11,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1699780928456" 2023-11-12T09:22:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1699780928456 2023-11-12T09:22:11,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster$12(2662): Client=jenkins//172.31.12.81 enable backup:system 2023-11-12T09:22:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=51, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-12T09:22:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=51 2023-11-12T09:22:11,204 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-12T09:22:11,204 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780931204"}]},"ts":"1699780931204"} 2023-11-12T09:22:11,206 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-12T09:22:11,208 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-12T09:22:11,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, ASSIGN}] 2023-11-12T09:22:11,210 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, ASSIGN 2023-11-12T09:22:11,211 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-12T09:22:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=51 2023-11-12T09:22:11,340 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-12T09:22:11,362 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=bf8e1ef1f6635e037c32e6a8e41782e2, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:11,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2, server=jenkins-hbase3.apache.org,44111,1699780893376}] 2023-11-12T09:22:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=51 2023-11-12T09:22:11,516 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:11,520 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(130): Open backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:11,520 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7254): Opening region: {ENCODED => bf8e1ef1f6635e037c32e6a8e41782e2, NAME => 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.', STARTKEY => '', ENDKEY => ''} 2023-11-12T09:22:11,521 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-12T09:22:11,521 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:11,521 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(888): Instantiated backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-12T09:22:11,521 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7296): checking encryption for bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:11,521 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7299): checking classloading for bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:11,523 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:11,524 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf8e1ef1f6635e037c32e6a8e41782e2 columnFamilyName meta 2023-11-12T09:22:11,524 DEBUG [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:22:11,530 DEBUG [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0 2023-11-12T09:22:11,530 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.HStore(324): Store=bf8e1ef1f6635e037c32e6a8e41782e2/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:22:11,530 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:11,531 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf8e1ef1f6635e037c32e6a8e41782e2 columnFamilyName session 2023-11-12T09:22:11,531 DEBUG [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-12T09:22:11,537 DEBUG [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c 2023-11-12T09:22:11,542 DEBUG [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0 2023-11-12T09:22:11,543 INFO [StoreOpener-bf8e1ef1f6635e037c32e6a8e41782e2-1 {}] regionserver.HStore(324): Store=bf8e1ef1f6635e037c32e6a8e41782e2/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-12T09:22:11,544 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:11,545 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:11,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-12T09:22:11,548 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1076): writing seq id for bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:11,549 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): Opened bf8e1ef1f6635e037c32e6a8e41782e2; next sequenceid=35; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10839198720, jitterRate=0.009479045867919922}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-12T09:22:11,549 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(992): Region open journal for bf8e1ef1f6635e037c32e6a8e41782e2: 2023-11-12T09:22:11,550 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2., pid=53, masterSystemTime=1699780931516 2023-11-12T09:22:11,552 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:11,552 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(158): Opened backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:11,552 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=bf8e1ef1f6635e037c32e6a8e41782e2, regionState=OPEN, openSeqNum=35, regionLocation=jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:11,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-11-12T09:22:11,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2, server=jenkins-hbase3.apache.org,44111,1699780893376 in 190 msec 2023-11-12T09:22:11,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=52, resume processing ppid=51 2023-11-12T09:22:11,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=bf8e1ef1f6635e037c32e6a8e41782e2, ASSIGN in 347 msec 2023-11-12T09:22:11,558 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1699780931557"}]},"ts":"1699780931557"} 2023-11-12T09:22:11,559 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-12T09:22:11,561 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-12T09:22:11,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, state=SUCCESS; EnableTableProcedure table=backup:system in 361 msec 2023-11-12T09:22:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=51 2023-11-12T09:22:11,807 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-12T09:22:11,807 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:22:11,824 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:22:11,837 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(128): Backup system table snapshot exists 2023-11-12T09:22:11,840 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(227): Deleting backup backup_1699780907622 ... 2023-11-12T09:22:11,843 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(445): Trying to cleanup up target dir : backup_1699780907622 2023-11-12T09:22:11,843 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(460): No data has been found in hdfs://localhost:37959/backupUT/backup_1699780907622/default/test-1699780901850. 2023-11-12T09:22:11,844 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(467): hdfs://localhost:37959/backupUT/backup_1699780907622/default is empty, remove it. 2023-11-12T09:22:11,849 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-11-12T09:22:11,850 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-11-12T09:22:11,851 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(276): Delete backup backup_1699780907622 completed. 2023-11-12T09:22:11,854 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1034): test-1699780901850 2023-11-12T09:22:11,857 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:22:11,865 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:22:11,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:22:11,868 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:22:11,868 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(657): Finish backup exclusive operation DELETE operation finished OK: backup_1699780907622 No failed backup MERGE operation found 2023-11-12T09:22:11,872 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:22:11,876 ERROR [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1579): Snapshot snapshot_backup_system does not exists 2023-11-12T09:22:11,877 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:11,877 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:11,882 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] backup.TestBackupDeleteWithFailures(143): test repair backup delete on a single table with data and failures POST_DELETE_SNAPSHOT_FAILURE 2023-11-12T09:22:11,889 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330418f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:11,893 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:11,895 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45798, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:22:11,897 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:22:11,899 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:41264, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:22:11,913 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:22:11,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699780931920 (current time:1699780931920). 2023-11-12T09:22:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:22:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:22:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459be524, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:11,936 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45800, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:22:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:22:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-11-12T09:22:11,941 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:22:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:22:11,942 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:22:11,945 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot, current working dir path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:22:11,948 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:22:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:22:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:22:12,358 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:22:12,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2}] 2023-11-12T09:22:12,361 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:12,512 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:12,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44111 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2023-11-12T09:22:12,513 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:12,514 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2819): Flushing bf8e1ef1f6635e037c32e6a8e41782e2 2/2 column families, dataSize=417 B heapSize=1.49 KB 2023-11-12T09:22:12,530 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/3b2b43463d364ec8acd53ea5b8cb0138 is 91, key is incrbackupset:hdfs://localhost:37959/backupUT/meta:test-1699780901850/1699780931854/Put/seqid=0 2023-11-12T09:22:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:22:12,936 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=184 B at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/3b2b43463d364ec8acd53ea5b8cb0138 2023-11-12T09:22:12,942 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3b2b43463d364ec8acd53ea5b8cb0138 2023-11-12T09:22:12,956 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/eee9e536d0ef4b33adcf42c561307f68 is 59, key is session:backup_1699780907622/session:/1699780931850/DeleteFamily/seqid=0 2023-11-12T09:22:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:22:13,360 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=233 B at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/eee9e536d0ef4b33adcf42c561307f68 2023-11-12T09:22:13,367 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eee9e536d0ef4b33adcf42c561307f68 2023-11-12T09:22:13,369 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/3b2b43463d364ec8acd53ea5b8cb0138 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/3b2b43463d364ec8acd53ea5b8cb0138 2023-11-12T09:22:13,375 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3b2b43463d364ec8acd53ea5b8cb0138 2023-11-12T09:22:13,375 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/3b2b43463d364ec8acd53ea5b8cb0138, entries=2, sequenceid=45, filesize=5.2 K 2023-11-12T09:22:13,376 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/eee9e536d0ef4b33adcf42c561307f68 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/eee9e536d0ef4b33adcf42c561307f68 2023-11-12T09:22:13,382 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eee9e536d0ef4b33adcf42c561307f68 2023-11-12T09:22:13,382 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/eee9e536d0ef4b33adcf42c561307f68, entries=2, sequenceid=45, filesize=5.2 K 2023-11-12T09:22:13,383 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(3022): Finished flush of dataSize ~417 B/417, heapSize ~1.46 KB/1496, currentSize=0 B/0 for bf8e1ef1f6635e037c32e6a8e41782e2 in 869ms, sequenceid=45, compaction requested=true 2023-11-12T09:22:13,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2520): Flush status journal for bf8e1ef1f6635e037c32e6a8e41782e2: 2023-11-12T09:22:13,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. for snapshot_backup_system completed. 2023-11-12T09:22:13,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:22:13,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:22:13,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/3b2b43463d364ec8acd53ea5b8cb0138] hfiles 2023-11-12T09:22:13,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0 for snapshot=snapshot_backup_system 2023-11-12T09:22:13,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/3b2b43463d364ec8acd53ea5b8cb0138 for snapshot=snapshot_backup_system 2023-11-12T09:22:13,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/eee9e536d0ef4b33adcf42c561307f68] hfiles 2023-11-12T09:22:13,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c for snapshot=snapshot_backup_system 2023-11-12T09:22:13,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0 for snapshot=snapshot_backup_system 2023-11-12T09:22:13,383 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/eee9e536d0ef4b33adcf42c561307f68 for snapshot=snapshot_backup_system 2023-11-12T09:22:13,792 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:13,792 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=55 2023-11-12T09:22:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster(4026): Remote procedure done, pid=55 2023-11-12T09:22:13,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:13,794 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:13,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-11-12T09:22:13,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 in 1.4360 sec 2023-11-12T09:22:13,796 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:22:13,797 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:22:13,798 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:22:13,798 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:22:13,799 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:22:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:22:14,214 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:22:14,223 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:22:14,224 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:22:14,225 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:22:14,225 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-11-12T09:22:14,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2870 sec 2023-11-12T09:22:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-12T09:22:16,051 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(115): Backup backup_1699780931890 started at 1699780936050. 2023-11-12T09:22:16,052 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(119): Backup session backup_1699780931890 has been started. 2023-11-12T09:22:16,054 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:22:16,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:22:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:22:16,055 INFO [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:22:16,055 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:22:16,056 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:22:16,056 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:22:16,057 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:22:16,058 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:16,058 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:22:16,058 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:22:16,059 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:22:16,059 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:22:16,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:22:16,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:22:16,060 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:22:16,060 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:22:16,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:22:16,060 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:22:16,060 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:22:16,061 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:22:16,061 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:22:16,061 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44111,1699780893376' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:22:16,064 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:22:16,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:16,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:22:16,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:22:16,065 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:22:16,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:22:16,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:22:16,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:22:16,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:16,067 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:22:16,067 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44111,1699780893376' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:22:16,067 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@36ba1a3[Count = 0] remaining members to acquire global barrier 2023-11-12T09:22:16,067 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:22:16,067 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:22:16,069 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:22:16,069 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:22:16,069 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:22:16,069 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:22:16,069 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:22:16,069 DEBUG [rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:16,069 INFO [rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699780913320 highest: 1699780913320 on jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:16,069 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta:.meta(num 1699780913291) roll requested 2023-11-12T09:22:16,081 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780936069.meta, exclude list is [], retry=0 2023-11-12T09:22:16,086 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK] 2023-11-12T09:22:16,088 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780913291.meta with entries=20, filesize=4.92 KB; new WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780936069.meta 2023-11-12T09:22:16,088 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK]] 2023-11-12T09:22:16,088 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780913291.meta is not closed yet, will try archiving it next time 2023-11-12T09:22:16,088 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44111%2C1699780893376:(num 1699780913320) roll requested 2023-11-12T09:22:16,090 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780913291.meta not finished, retry = 0 2023-11-12T09:22:16,100 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780936088, exclude list is [], retry=0 2023-11-12T09:22:16,104 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK] 2023-11-12T09:22:16,105 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780913320 with entries=35, filesize=7.33 KB; new WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780936088 2023-11-12T09:22:16,106 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK]] 2023-11-12T09:22:16,106 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780913320 is not closed yet, will try archiving it next time 2023-11-12T09:22:16,109 DEBUG [rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:22:16,109 INFO [rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699780936088 on jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:16,120 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:22:16,120 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44111,1699780893376' in zk 2023-11-12T09:22:16,123 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:22:16,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:16,123 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:22:16,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:22:16,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:22:16,123 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:22:16,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:22:16,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:22:16,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:22:16,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:16,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:22:16,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:22:16,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:16,127 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44111,1699780893376': 2023-11-12T09:22:16,127 INFO [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:22:16,127 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44111,1699780893376' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:22:16,127 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:22:16,127 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:22:16,127 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:22:16,127 INFO [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:22:16,129 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:22:16,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:22:16,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:22:16,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:22:16,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:22:16,130 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:22:16,130 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:22:16,130 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:22:16,130 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:22:16,130 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:22:16,131 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:22:16,131 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:22:16,132 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:16,132 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:22:16,133 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:22:16,133 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:16,138 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:22:16,138 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:22:16,138 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:22:16,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:22:16,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:22:16,140 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:22:16,140 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:22:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:22:16,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } 2023-11-12T09:22:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699780936243 (current time:1699780936243). 2023-11-12T09:22:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:22:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699780936242_default_test-1699780901850 VERSION not specified, setting to 2 2023-11-12T09:22:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73e99ef9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:16,254 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:53056, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:22:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:22:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } 2023-11-12T09:22:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 }, snapshot procedure id = 56 2023-11-12T09:22:16,258 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:22:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-12T09:22:16,259 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:22:16,261 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot, current working dir path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_1699780936242_default_test-1699780901850 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:22:16,264 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:22:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-12T09:22:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-12T09:22:16,670 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:22:16,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE; SnapshotRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35}] 2023-11-12T09:22:16,673 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=57, ppid=56, state=RUNNABLE; SnapshotRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:22:16,824 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:16,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44111 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2023-11-12T09:22:16,824 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:22:16,825 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2520): Flush status journal for 9d3fa1e74299c566cf2ed2c7ca01ec35: 2023-11-12T09:22:16,825 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. for snapshot_1699780936242_default_test-1699780901850 completed. 2023-11-12T09:22:16,825 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(240): Storing 'test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35.' region-info for snapshot=snapshot_1699780936242_default_test-1699780901850 2023-11-12T09:22:16,825 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:22:16,825 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc] hfiles 2023-11-12T09:22:16,825 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc for snapshot=snapshot_1699780936242_default_test-1699780901850 2023-11-12T09:22:16,843 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-12T09:22:16,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-12T09:22:17,234 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:22:17,234 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=57 2023-11-12T09:22:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster(4026): Remote procedure done, pid=57 2023-11-12T09:22:17,235 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699780936242_default_test-1699780901850 on region 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:22:17,236 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=57, ppid=56, state=RUNNABLE; SnapshotRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:22:17,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=57, resume processing ppid=56 2023-11-12T09:22:17,239 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:22:17,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, ppid=56, state=SUCCESS; SnapshotRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35 in 567 msec 2023-11-12T09:22:17,240 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:22:17,241 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:22:17,241 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699780936242_default_test-1699780901850 2023-11-12T09:22:17,241 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_1699780936242_default_test-1699780901850 2023-11-12T09:22:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-12T09:22:17,652 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:22:17,658 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:22:17,659 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_1699780936242_default_test-1699780901850 to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_1699780936242_default_test-1699780901850 2023-11-12T09:22:17,660 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:22:17,660 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 }, snapshot procedure id = 56 2023-11-12T09:22:17,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1699780936242_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } in 1.4060 sec 2023-11-12T09:22:18,066 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-11-12T09:22:18,066 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-11-12T09:22:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-12T09:22:18,364 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699780931890 2023-11-12T09:22:18,364 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:22:18,364 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-12T09:22:18,364 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699780931890_test-1699780901850 2023-11-12T09:22:18,365 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699780936242_default_test-1699780901850 to hdfs://localhost:37959/backupUT/backup_1699780931890/default/test-1699780901850/ 2023-11-12T09:22:18,365 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:22:18,389 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37959, inputRoot=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe 2023-11-12T09:22:18,389 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33463,1699780890205_1071104246_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37959/backupUT/backup_1699780931890/default/test-1699780901850, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37959/backupUT/backup_1699780931890/default/test-1699780901850/.hbase-snapshot/.tmp/snapshot_1699780936242_default_test-1699780901850 2023-11-12T09:22:18,389 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37959, inputRoot=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe, snapshotDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_1699780936242_default_test-1699780901850. 2023-11-12T09:22:18,397 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_1699780936242_default_test-1699780901850 to hdfs://localhost:37959/backupUT/backup_1699780931890/default/test-1699780901850/.hbase-snapshot/.tmp/snapshot_1699780936242_default_test-1699780901850 2023-11-12T09:22:18,810 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:18,811 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:18,812 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:18,812 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:19,796 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/hadoop-7389803058672189676.jar 2023-11-12T09:22:19,796 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:19,797 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:19,797 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:19,798 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:19,798 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:19,799 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:19,799 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:19,799 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:19,800 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:19,801 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:22:19,801 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:22:19,802 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:22:19,802 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:22:19,803 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:22:19,803 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:22:19,803 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:22:19,804 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:22:19,804 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:22:19,805 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:22:19,805 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:22:19,806 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:22:19,806 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:22:19,807 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:22:19,808 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:22:19,808 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:22:19,808 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:22:19,809 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:22:19,810 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:22:19,854 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699780936242_default_test-1699780901850' hfile list 2023-11-12T09:22:19,856 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-12T09:22:19,932 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/mapreduce.cluster.local.dir/1699780939897/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:22:19,932 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/mapreduce.cluster.local.dir/1699780939897/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:22:19,989 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:22:20,037 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc output=hdfs://localhost:37959/backupUT/backup_1699780931890/default/test-1699780901850/archive/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc 2023-11-12T09:22:20,038 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-11-12T09:22:20,973 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:22:20,974 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:22:20,980 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699780936242_default_test-1699780901850 2023-11-12T09:22:20,980 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699780936242_default_test-1699780901850 finished. 2023-11-12T09:22:20,981 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1034): test-1699780901850 2023-11-12T09:22:20,988 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699780931890 2023-11-12T09:22:20,988 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:22:21,340 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-12T09:22:21,340 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-11-12T09:22:21,394 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37959/backupUT/backup_1699780931890/.backup.manifest 2023-11-12T09:22:21,394 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(358): Backup backup_1699780931890 finished: type=FULL,tablelist=test-1699780901850,targetRootDir=hdfs://localhost:37959/backupUT,startts=1699780936050,completets=1699780940988,bytescopied=0 2023-11-12T09:22:21,394 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:22:21,394 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699780936242_default_test-1699780901850 2023-11-12T09:22:21,395 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699780936242_default_test-1699780901850" 2023-11-12T09:22:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699780936242_default_test-1699780901850 2023-11-12T09:22:21,399 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699780936242_default_test-1699780901850 for backup backup_1699780931890 succeeded. 2023-11-12T09:22:21,399 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:22:21,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:22:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:22:21,409 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:22:21,411 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:22:21,417 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(376): Backup backup_1699780931890 completed. 2023-11-12T09:22:21,417 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:21,430 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] backup.TestBackupDeleteWithFailures(147): backup complete 2023-11-12T09:22:21,450 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:22:21,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699780941460 (current time:1699780941460). 2023-11-12T09:22:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:22:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:22:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d896829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:21,471 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:53060, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:22:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:22:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 58 2023-11-12T09:22:21,475 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:22:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2023-11-12T09:22:21,476 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:22:21,478 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot, current working dir path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:22:21,481 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:22:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2023-11-12T09:22:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2023-11-12T09:22:21,889 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:22:21,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2}] 2023-11-12T09:22:21,891 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=59, ppid=58, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:22,042 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:22,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44111 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2023-11-12T09:22:22,043 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:22,043 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2819): Flushing bf8e1ef1f6635e037c32e6a8e41782e2 2/2 column families, dataSize=1.06 KB heapSize=2.22 KB 2023-11-12T09:22:22,047 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/f1dd90e0369443adb4bc8b51a1e4413c is 169, key is trslm:hdfs://localhost:37959/backupUT\x00test-1699780901850/meta:log-roll-map/1699780940984/Put/seqid=0 2023-11-12T09:22:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2023-11-12T09:22:22,453 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=511 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/f1dd90e0369443adb4bc8b51a1e4413c 2023-11-12T09:22:22,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/0baa235d98fc45fbbbed151f8c19fc12 is 310, key is session:backup_1699780931890/session:context/1699780941410/Put/seqid=0 2023-11-12T09:22:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2023-11-12T09:22:22,866 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/0baa235d98fc45fbbbed151f8c19fc12 2023-11-12T09:22:22,873 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/f1dd90e0369443adb4bc8b51a1e4413c as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/f1dd90e0369443adb4bc8b51a1e4413c 2023-11-12T09:22:22,879 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/f1dd90e0369443adb4bc8b51a1e4413c, entries=5, sequenceid=57, filesize=5.5 K 2023-11-12T09:22:22,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/0baa235d98fc45fbbbed151f8c19fc12 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0baa235d98fc45fbbbed151f8c19fc12 2023-11-12T09:22:22,885 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0baa235d98fc45fbbbed151f8c19fc12, entries=2, sequenceid=57, filesize=5.3 K 2023-11-12T09:22:22,886 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(3022): Finished flush of dataSize ~1.06 KB/1090, heapSize ~2.19 KB/2240, currentSize=0 B/0 for bf8e1ef1f6635e037c32e6a8e41782e2 in 843ms, sequenceid=57, compaction requested=true 2023-11-12T09:22:22,886 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2520): Flush status journal for bf8e1ef1f6635e037c32e6a8e41782e2: 2023-11-12T09:22:22,886 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. for snapshot_backup_system completed. 2023-11-12T09:22:22,886 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:22:22,886 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:22:22,886 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/3b2b43463d364ec8acd53ea5b8cb0138, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/f1dd90e0369443adb4bc8b51a1e4413c] hfiles 2023-11-12T09:22:22,886 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0 for snapshot=snapshot_backup_system 2023-11-12T09:22:22,886 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/3b2b43463d364ec8acd53ea5b8cb0138 for snapshot=snapshot_backup_system 2023-11-12T09:22:22,886 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/f1dd90e0369443adb4bc8b51a1e4413c for snapshot=snapshot_backup_system 2023-11-12T09:22:22,886 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/eee9e536d0ef4b33adcf42c561307f68, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0baa235d98fc45fbbbed151f8c19fc12] hfiles 2023-11-12T09:22:22,886 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c for snapshot=snapshot_backup_system 2023-11-12T09:22:22,886 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0 for snapshot=snapshot_backup_system 2023-11-12T09:22:22,886 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/eee9e536d0ef4b33adcf42c561307f68 for snapshot=snapshot_backup_system 2023-11-12T09:22:22,886 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0baa235d98fc45fbbbed151f8c19fc12 for snapshot=snapshot_backup_system 2023-11-12T09:22:23,294 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:23,294 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=59 2023-11-12T09:22:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster(4026): Remote procedure done, pid=59 2023-11-12T09:22:23,294 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:23,296 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=59, ppid=58, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:23,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=59, resume processing ppid=58 2023-11-12T09:22:23,298 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:22:23,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, ppid=58, state=SUCCESS; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 in 1.4080 sec 2023-11-12T09:22:23,299 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:22:23,299 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:22:23,300 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:22:23,300 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:22:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2023-11-12T09:22:23,710 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:22:23,719 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:22:23,719 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:22:23,720 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:22:23,720 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 58 2023-11-12T09:22:23,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2480 sec 2023-11-12T09:22:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2023-11-12T09:22:25,589 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(227): Deleting backup backup_1699780931890 ... 2023-11-12T09:22:25,590 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(437): Delete log files: .backup.manifest 2023-11-12T09:22:25,591 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(437): Delete log files: default 2023-11-12T09:22:25,592 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(445): Trying to cleanup up target dir : backup_1699780931890 2023-11-12T09:22:25,593 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(460): No data has been found in hdfs://localhost:37959/backupUT/backup_1699780931890/default/test-1699780901850. 2023-11-12T09:22:25,594 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(467): hdfs://localhost:37959/backupUT/backup_1699780931890/default is empty, remove it. 2023-11-12T09:22:25,598 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-11-12T09:22:25,598 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-11-12T09:22:25,599 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(276): Delete backup backup_1699780931890 completed. 2023-11-12T09:22:25,608 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1034): test-1699780901850 2023-11-12T09:22:25,610 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:22:25,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:22:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:22:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: postDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.postDeleteSnapshot(TestBackupDeleteWithFailures.java:111) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$93.call(MasterCoprocessorHost.java:1218) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$93.call(MasterCoprocessorHost.java:1215) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.postDeleteSnapshot(MasterCoprocessorHost.java:1215) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:388) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-12T09:22:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.CallRunner(138): callId: 282 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:52850 deadline: 1699781005618, exception=java.io.IOException: postDeleteSnapshot 2023-11-12T09:22:25,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:22:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:361) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-12T09:22:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.CallRunner(138): callId: 284 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:52850 deadline: 1699781005724, exception=org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem 2023-11-12T09:22:25,730 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(163): Delete operation succeeded, there were some errors: org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:361) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:397) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:385) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.MasterCallable.call(MasterCallable.java:101) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3173) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3165) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.deleteSnapshot(HBaseAdmin.java:3073) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.backup.impl.BackupSystemTable.deleteSnapshot(BackupSystemTable.java:1576) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.deleteBackups(BackupAdminImpl.java:149) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailuresAfter(TestBackupDeleteWithFailures.java:161) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailures(TestBackupDeleteWithFailures.java:137) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:361) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:764) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:199) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:220) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] ... 1 more 2023-11-12T09:22:25,732 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:22:25,746 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@166f1c71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:25,752 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71edbbe5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:25,753 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:22:25,754 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36868, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:22:25,758 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:25,759 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45470, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService REPAIR status: no failed sessions found. Checking failed delete backup operation ... No failed backup DELETE operation found 2023-11-12T09:22:25,782 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:22:25,786 ERROR [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1579): Snapshot snapshot_backup_system does not exists No failed backup MERGE operation found 2023-11-12T09:22:25,787 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:22:25,791 ERROR [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1579): Snapshot snapshot_backup_system does not exists 2023-11-12T09:22:25,792 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:25,792 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:25,798 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] backup.TestBackupDeleteWithFailures(143): test repair backup delete on a single table with data and failures PRE_SNAPSHOT_FAILURE 2023-11-12T09:22:25,813 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aff8097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:25,817 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:25,819 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45474, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:22:25,820 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:22:25,821 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36870, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:22:25,834 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:22:25,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699780945839 (current time:1699780945839). 2023-11-12T09:22:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:22:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:22:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@412dddde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:25,847 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45486, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:22:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:22:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-11-12T09:22:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:22:25,854 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:22:25,854 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:22:25,857 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot, current working dir path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:22:25,860 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:22:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:22:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:22:26,266 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:22:26,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2}] 2023-11-12T09:22:26,268 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:26,419 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:26,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44111 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2023-11-12T09:22:26,420 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:26,420 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2819): Flushing bf8e1ef1f6635e037c32e6a8e41782e2 2/2 column families, dataSize=268 B heapSize=1.13 KB 2023-11-12T09:22:26,424 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/b5d8d76133d6403bb2b468b777214da4 is 91, key is incrbackupset:hdfs://localhost:37959/backupUT/meta:test-1699780901850/1699780945609/Put/seqid=0 2023-11-12T09:22:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:22:26,832 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/b5d8d76133d6403bb2b468b777214da4 2023-11-12T09:22:26,837 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b5d8d76133d6403bb2b468b777214da4 2023-11-12T09:22:26,841 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/f1b020543a4c4efcbadc4cbf6467e527 is 59, key is session:backup_1699780931890/session:/1699780945598/DeleteFamily/seqid=0 2023-11-12T09:22:26,847 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=144 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/f1b020543a4c4efcbadc4cbf6467e527 2023-11-12T09:22:26,853 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f1b020543a4c4efcbadc4cbf6467e527 2023-11-12T09:22:26,855 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/b5d8d76133d6403bb2b468b777214da4 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/b5d8d76133d6403bb2b468b777214da4 2023-11-12T09:22:26,862 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b5d8d76133d6403bb2b468b777214da4 2023-11-12T09:22:26,862 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/b5d8d76133d6403bb2b468b777214da4, entries=2, sequenceid=65, filesize=5.2 K 2023-11-12T09:22:26,863 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/f1b020543a4c4efcbadc4cbf6467e527 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/f1b020543a4c4efcbadc4cbf6467e527 2023-11-12T09:22:26,870 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f1b020543a4c4efcbadc4cbf6467e527 2023-11-12T09:22:26,870 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/f1b020543a4c4efcbadc4cbf6467e527, entries=2, sequenceid=65, filesize=5.2 K 2023-11-12T09:22:26,871 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3022): Finished flush of dataSize ~268 B/268, heapSize ~1.09 KB/1120, currentSize=0 B/0 for bf8e1ef1f6635e037c32e6a8e41782e2 in 451ms, sequenceid=65, compaction requested=true 2023-11-12T09:22:26,871 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2520): Flush status journal for bf8e1ef1f6635e037c32e6a8e41782e2: 2023-11-12T09:22:26,871 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. for snapshot_backup_system completed. 2023-11-12T09:22:26,871 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:22:26,871 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:22:26,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/3b2b43463d364ec8acd53ea5b8cb0138, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/f1dd90e0369443adb4bc8b51a1e4413c, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/b5d8d76133d6403bb2b468b777214da4] hfiles 2023-11-12T09:22:26,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0 for snapshot=snapshot_backup_system 2023-11-12T09:22:26,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/3b2b43463d364ec8acd53ea5b8cb0138 for snapshot=snapshot_backup_system 2023-11-12T09:22:26,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/f1dd90e0369443adb4bc8b51a1e4413c for snapshot=snapshot_backup_system 2023-11-12T09:22:26,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/b5d8d76133d6403bb2b468b777214da4 for snapshot=snapshot_backup_system 2023-11-12T09:22:26,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/eee9e536d0ef4b33adcf42c561307f68, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0baa235d98fc45fbbbed151f8c19fc12, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/f1b020543a4c4efcbadc4cbf6467e527] hfiles 2023-11-12T09:22:26,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c for snapshot=snapshot_backup_system 2023-11-12T09:22:26,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0 for snapshot=snapshot_backup_system 2023-11-12T09:22:26,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/eee9e536d0ef4b33adcf42c561307f68 for snapshot=snapshot_backup_system 2023-11-12T09:22:26,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0baa235d98fc45fbbbed151f8c19fc12 for snapshot=snapshot_backup_system 2023-11-12T09:22:26,872 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/f1b020543a4c4efcbadc4cbf6467e527 for snapshot=snapshot_backup_system 2023-11-12T09:22:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:22:27,277 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:27,277 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=61 2023-11-12T09:22:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster(4026): Remote procedure done, pid=61 2023-11-12T09:22:27,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:27,279 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:27,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-11-12T09:22:27,281 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:22:27,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 in 1.0140 sec 2023-11-12T09:22:27,282 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:22:27,283 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:22:27,283 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:22:27,283 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:22:27,694 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:22:27,708 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:22:27,709 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:22:27,711 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:22:27,711 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-11-12T09:22:27,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8620 sec 2023-11-12T09:22:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-12T09:22:27,960 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(115): Backup backup_1699780945813 started at 1699780947960. 2023-11-12T09:22:27,962 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(119): Backup session backup_1699780945813 has been started. 2023-11-12T09:22:27,964 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-12T09:22:27,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(962): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-11-12T09:22:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-12T09:22:27,965 INFO [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-12T09:22:27,965 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-12T09:22:27,966 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-12T09:22:27,966 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-12T09:22:27,967 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:22:27,968 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:27,969 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:22:27,969 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:22:27,969 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-12T09:22:27,969 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-12T09:22:27,970 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-12T09:22:27,970 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-12T09:22:27,970 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-12T09:22:27,970 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-12T09:22:27,971 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-12T09:22:27,971 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-12T09:22:27,971 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-12T09:22:27,972 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-12T09:22:27,972 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-12T09:22:27,972 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,44111,1699780893376' joining acquired barrier for procedure (rolllog) in zk 2023-11-12T09:22:27,975 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-12T09:22:27,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:27,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:22:27,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:22:27,975 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-12T09:22:27,975 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:22:27,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:22:27,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:22:27,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:27,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:22:27,977 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,44111,1699780893376' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-12T09:22:27,977 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@36bedb08[Count = 0] remaining members to acquire global barrier 2023-11-12T09:22:27,977 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-12T09:22:27,977 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-12T09:22:27,979 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-12T09:22:27,979 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-12T09:22:27,979 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-12T09:22:27,979 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-12T09:22:27,981 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-12T09:22:27,981 DEBUG [rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:27,981 INFO [rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1699780936088 highest: 1699780936088 on jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:27,981 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta:.meta(num 1699780936069) roll requested 2023-11-12T09:22:27,993 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780947981.meta, exclude list is [], retry=0 2023-11-12T09:22:27,997 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK] 2023-11-12T09:22:28,000 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780936069.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780947981.meta 2023-11-12T09:22:28,000 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK]] 2023-11-12T09:22:28,000 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780936069.meta is not closed yet, will try archiving it next time 2023-11-12T09:22:28,000 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C44111%2C1699780893376:(num 1699780936088) roll requested 2023-11-12T09:22:28,002 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780936069.meta not finished, retry = 0 2023-11-12T09:22:28,016 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780948000, exclude list is [], retry=0 2023-11-12T09:22:28,020 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK] 2023-11-12T09:22:28,021 DEBUG [rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-12T09:22:28,021 INFO [rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1699780948000 on jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:28,022 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780936088 with entries=18, filesize=3.44 KB; new WAL /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780948000 2023-11-12T09:22:28,022 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41591,DS-2ab59756-3d8b-485f-95c2-dd011ce9885e,DISK]] 2023-11-12T09:22:28,022 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780936088 is not closed yet, will try archiving it next time 2023-11-12T09:22:28,024 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780913320 to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/oldWALs/jenkins-hbase3.apache.org%2C44111%2C1699780893376.1699780913320 2023-11-12T09:22:28,032 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-12T09:22:28,032 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,44111,1699780893376' in zk 2023-11-12T09:22:28,036 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-12T09:22:28,036 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:22:28,036 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:28,036 DEBUG [member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-12T09:22:28,036 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:22:28,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:22:28,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:22:28,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:22:28,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:22:28,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:28,039 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:22:28,039 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:22:28,040 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:28,040 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,44111,1699780893376': 2023-11-12T09:22:28,040 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,44111,1699780893376' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-12T09:22:28,041 INFO [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-12T09:22:28,041 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-12T09:22:28,041 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-12T09:22:28,041 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-12T09:22:28,041 INFO [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-12T09:22:28,043 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-12T09:22:28,043 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-12T09:22:28,043 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:22:28,043 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-12T09:22:28,043 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-12T09:22:28,044 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:22:28,044 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:22:28,044 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-12T09:22:28,044 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-12T09:22:28,044 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:22:28,045 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-12T09:22:28,045 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:22:28,046 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:28,046 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-12T09:22:28,047 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-12T09:22:28,047 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:28,051 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-12T09:22:28,051 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-12T09:22:28,051 DEBUG [(jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-12T09:22:28,051 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-12T09:22:28,051 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-12T09:22:28,052 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-12T09:22:28,052 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-12T09:22:28,104 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/WALs/jenkins-hbase3.apache.org,44111,1699780893376/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780936069.meta to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/oldWALs/jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta.1699780936069.meta 2023-11-12T09:22:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-12T09:22:28,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } 2023-11-12T09:22:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699780948156 (current time:1699780948156). 2023-11-12T09:22:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:22:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1699780948156_default_test-1699780901850 VERSION not specified, setting to 2 2023-11-12T09:22:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5b4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:28,172 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45494, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:22:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:22:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } 2023-11-12T09:22:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-11-12T09:22:28,176 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:22:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-12T09:22:28,177 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:22:28,180 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot, current working dir path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_1699780948156_default_test-1699780901850 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:22:28,182 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:22:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-12T09:22:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-12T09:22:28,589 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:22:28,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35}] 2023-11-12T09:22:28,594 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:22:28,746 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:28,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44111 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-11-12T09:22:28,747 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:22:28,747 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for 9d3fa1e74299c566cf2ed2c7ca01ec35: 2023-11-12T09:22:28,747 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. for snapshot_1699780948156_default_test-1699780901850 completed. 2023-11-12T09:22:28,747 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35.' region-info for snapshot=snapshot_1699780948156_default_test-1699780901850 2023-11-12T09:22:28,747 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:22:28,748 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc] hfiles 2023-11-12T09:22:28,748 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc for snapshot=snapshot_1699780948156_default_test-1699780901850 2023-11-12T09:22:28,756 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:22:28,756 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-11-12T09:22:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster(4026): Remote procedure done, pid=63 2023-11-12T09:22:28,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1699780948156_default_test-1699780901850 on region 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:22:28,759 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:22:28,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-11-12T09:22:28,762 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:22:28,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure 9d3fa1e74299c566cf2ed2c7ca01ec35 in 171 msec 2023-11-12T09:22:28,763 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:22:28,763 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:22:28,763 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1699780948156_default_test-1699780901850 2023-11-12T09:22:28,764 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_1699780948156_default_test-1699780901850 2023-11-12T09:22:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-12T09:22:29,174 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:22:29,181 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:22:29,181 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_1699780948156_default_test-1699780901850 to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_1699780948156_default_test-1699780901850 2023-11-12T09:22:29,182 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:22:29,183 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-11-12T09:22:29,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1699780948156_default_test-1699780901850 table=test-1699780901850 type=FLUSH ttl=0 } in 1.0100 sec 2023-11-12T09:22:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-12T09:22:29,283 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1699780945813 2023-11-12T09:22:29,283 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-12T09:22:29,283 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-12T09:22:29,283 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1699780945813_test-1699780901850 2023-11-12T09:22:29,283 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1699780948156_default_test-1699780901850 to hdfs://localhost:37959/backupUT/backup_1699780945813/default/test-1699780901850/ 2023-11-12T09:22:29,283 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-12T09:22:29,307 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37959, inputRoot=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe 2023-11-12T09:22:29,307 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,33463,1699780890205_1166656997_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37959/backupUT/backup_1699780945813/default/test-1699780901850, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37959/backupUT/backup_1699780945813/default/test-1699780901850/.hbase-snapshot/.tmp/snapshot_1699780948156_default_test-1699780901850 2023-11-12T09:22:29,307 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37959, inputRoot=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe, snapshotDir=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_1699780948156_default_test-1699780901850. 2023-11-12T09:22:29,319 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_1699780948156_default_test-1699780901850 to hdfs://localhost:37959/backupUT/backup_1699780945813/default/test-1699780901850/.hbase-snapshot/.tmp/snapshot_1699780948156_default_test-1699780901850 2023-11-12T09:22:29,740 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:29,740 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:29,741 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:29,741 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:30,813 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/hadoop-3892828364426822739.jar 2023-11-12T09:22:30,814 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:30,814 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:30,815 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:30,815 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:30,816 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:30,816 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:30,816 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:30,817 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:30,817 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-12T09:22:30,818 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-12T09:22:30,819 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-12T09:22:30,819 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-12T09:22:30,820 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-12T09:22:30,820 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-12T09:22:30,821 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-12T09:22:30,822 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-12T09:22:30,822 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-12T09:22:30,823 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-12T09:22:30,823 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-12T09:22:30,824 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-12T09:22:30,824 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-12T09:22:30,825 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:22:30,826 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:22:30,827 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:22:30,827 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:22:30,828 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-12T09:22:30,828 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:22:30,829 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-12T09:22:30,877 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1699780948156_default_test-1699780901850' hfile list 2023-11-12T09:22:30,880 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-12T09:22:30,961 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/mapreduce.cluster.local.dir/1699780950924/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-12T09:22:30,961 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/mapreduce.cluster.local.dir/1699780950924/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-12T09:22:31,015 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-12T09:22:31,044 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc output=hdfs://localhost:37959/backupUT/backup_1699780945813/default/test-1699780901850/archive/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/f/366cfc2b3047493493ca939e4ba045cc 2023-11-12T09:22:31,045 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-11-12T09:22:31,996 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-12T09:22:31,998 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-12T09:22:32,005 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1699780948156_default_test-1699780901850 2023-11-12T09:22:32,005 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1699780948156_default_test-1699780901850 finished. 2023-11-12T09:22:32,005 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1034): test-1699780901850 2023-11-12T09:22:32,012 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1699780945813 2023-11-12T09:22:32,012 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-12T09:22:32,418 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37959/backupUT/backup_1699780945813/.backup.manifest 2023-11-12T09:22:32,418 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(358): Backup backup_1699780945813 finished: type=FULL,tablelist=test-1699780901850,targetRootDir=hdfs://localhost:37959/backupUT,startts=1699780947960,completets=1699780952012,bytescopied=0 2023-11-12T09:22:32,418 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-12T09:22:32,418 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1699780948156_default_test-1699780901850 2023-11-12T09:22:32,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_1699780948156_default_test-1699780901850" 2023-11-12T09:22:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1699780948156_default_test-1699780901850 2023-11-12T09:22:32,423 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1699780948156_default_test-1699780901850 for backup backup_1699780945813 succeeded. 2023-11-12T09:22:32,424 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:22:32,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:22:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:22:32,433 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:22:32,435 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:22:32,441 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.TableBackupClient(376): Backup backup_1699780945813 completed. 2023-11-12T09:22:32,441 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:32,455 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] backup.TestBackupDeleteWithFailures(147): backup complete 2023-11-12T09:22:32,477 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:22:32,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699780952487 (current time:1699780952487). 2023-11-12T09:22:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:22:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:22:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4549a091, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:32,497 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45502, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:22:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:32,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:22:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-11-12T09:22:32,501 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:22:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:22:32,502 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:22:32,504 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot, current working dir path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:22:32,506 WARN [PEWorker-1 {}] procedure.SnapshotProcedure(200): unexpected exception while execute pid=64, state=FAILED:SNAPSHOT_PRE_OPERATION, locked=true, exception=java.io.IOException via master-snapshot:java.io.IOException: preSnapshot; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }. Mark procedure Failed. java.io.IOException: preSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preSnapshot(TestBackupDeleteWithFailures.java:95) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$83.call(MasterCoprocessorHost.java:1122) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$83.call(MasterCoprocessorHost.java:1119) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preSnapshot(MasterCoprocessorHost.java:1119) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:306) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] 2023-11-12T09:22:32,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1531): Rolled back pid=64, state=ROLLEDBACK, exception=java.io.IOException via master-snapshot:java.io.IOException: preSnapshot; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } exec-time=23 msec 2023-11-12T09:22:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-12T09:22:32,618 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55b29a1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:32,623 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f99dba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:32,623 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-12T09:22:32,625 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:36882, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-12T09:22:32,628 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:32,630 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45508, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService REPAIR status: no failed sessions found. Checking failed delete backup operation ... Found failed DELETE operation for: backup_1699780945813 Running DELETE again ... 2023-11-12T09:22:32,644 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-12T09:22:32,648 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1550): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2023-11-12T09:22:32,648 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-12T09:22:32,659 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-12T09:22:32,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1699780952668 (current time:1699780952668). 2023-11-12T09:22:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-12T09:22:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-12T09:22:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113077fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-12T09:22:32,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-12T09:22:32,678 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.12.81:45514, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-12T09:22:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-12T09:22:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure2.ProcedureExecutor(1032): Stored pid=65, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-12T09:22:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 65 2023-11-12T09:22:32,681 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-12T09:22:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=65 2023-11-12T09:22:32,682 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-12T09:22:32,685 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot, current working dir path: hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-12T09:22:32,687 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-12T09:22:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=65 2023-11-12T09:22:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=65 2023-11-12T09:22:33,093 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-12T09:22:33,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2}] 2023-11-12T09:22:33,095 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=66, ppid=65, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:33,246 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:33,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44111 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=66 2023-11-12T09:22:33,247 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:33,247 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegion(2819): Flushing bf8e1ef1f6635e037c32e6a8e41782e2 2/2 column families, dataSize=1.21 KB heapSize=2.59 KB 2023-11-12T09:22:33,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/0c358e4967f74b08969fe0c7c6432efc is 169, key is trslm:hdfs://localhost:37959/backupUT\x00test-1699780901850/meta:log-roll-map/1699780952008/Put/seqid=0 2023-11-12T09:22:33,276 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=571 B at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/0c358e4967f74b08969fe0c7c6432efc 2023-11-12T09:22:33,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=65 2023-11-12T09:22:33,288 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/074fe361159b4cf9b29c50b7527dad4d is 310, key is session:backup_1699780945813/session:context/1699780952434/Put/seqid=0 2023-11-12T09:22:33,693 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=668 B at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/074fe361159b4cf9b29c50b7527dad4d 2023-11-12T09:22:33,700 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/0c358e4967f74b08969fe0c7c6432efc as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/0c358e4967f74b08969fe0c7c6432efc 2023-11-12T09:22:33,705 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/0c358e4967f74b08969fe0c7c6432efc, entries=5, sequenceid=80, filesize=5.5 K 2023-11-12T09:22:33,706 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/074fe361159b4cf9b29c50b7527dad4d as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/074fe361159b4cf9b29c50b7527dad4d 2023-11-12T09:22:33,712 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/074fe361159b4cf9b29c50b7527dad4d, entries=2, sequenceid=80, filesize=5.3 K 2023-11-12T09:22:33,713 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegion(3022): Finished flush of dataSize ~1.21 KB/1239, heapSize ~2.55 KB/2616, currentSize=0 B/0 for bf8e1ef1f6635e037c32e6a8e41782e2 in 466ms, sequenceid=80, compaction requested=true 2023-11-12T09:22:33,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegion(2520): Flush status journal for bf8e1ef1f6635e037c32e6a8e41782e2: 2023-11-12T09:22:33,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. for snapshot_backup_system completed. 2023-11-12T09:22:33,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2.' region-info for snapshot=snapshot_backup_system 2023-11-12T09:22:33,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-12T09:22:33,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/3b2b43463d364ec8acd53ea5b8cb0138, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/f1dd90e0369443adb4bc8b51a1e4413c, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/b5d8d76133d6403bb2b468b777214da4, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/0c358e4967f74b08969fe0c7c6432efc] hfiles 2023-11-12T09:22:33,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/442ef22cf4f0499c91e911cb5d9a70f0 for snapshot=snapshot_backup_system 2023-11-12T09:22:33,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/3b2b43463d364ec8acd53ea5b8cb0138 for snapshot=snapshot_backup_system 2023-11-12T09:22:33,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/f1dd90e0369443adb4bc8b51a1e4413c for snapshot=snapshot_backup_system 2023-11-12T09:22:33,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/b5d8d76133d6403bb2b468b777214da4 for snapshot=snapshot_backup_system 2023-11-12T09:22:33,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/0c358e4967f74b08969fe0c7c6432efc for snapshot=snapshot_backup_system 2023-11-12T09:22:33,713 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/eee9e536d0ef4b33adcf42c561307f68, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0baa235d98fc45fbbbed151f8c19fc12, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/f1b020543a4c4efcbadc4cbf6467e527, hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/074fe361159b4cf9b29c50b7527dad4d] hfiles 2023-11-12T09:22:33,714 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0385264caa2440b4b46b20a5b930cc8c for snapshot=snapshot_backup_system 2023-11-12T09:22:33,714 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/efa79550ed004f0c9565636ed190b0d0 for snapshot=snapshot_backup_system 2023-11-12T09:22:33,714 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/eee9e536d0ef4b33adcf42c561307f68 for snapshot=snapshot_backup_system 2023-11-12T09:22:33,714 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/0baa235d98fc45fbbbed151f8c19fc12 for snapshot=snapshot_backup_system 2023-11-12T09:22:33,714 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/f1b020543a4c4efcbadc4cbf6467e527 for snapshot=snapshot_backup_system 2023-11-12T09:22:33,714 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/074fe361159b4cf9b29c50b7527dad4d for snapshot=snapshot_backup_system 2023-11-12T09:22:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=65 2023-11-12T09:22:34,120 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:34,120 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=66 2023-11-12T09:22:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.HMaster(4026): Remote procedure done, pid=66 2023-11-12T09:22:34,121 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:34,122 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=66, ppid=65, state=RUNNABLE; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:34,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=66, resume processing ppid=65 2023-11-12T09:22:34,124 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-12T09:22:34,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, ppid=65, state=SUCCESS; SnapshotRegionProcedure bf8e1ef1f6635e037c32e6a8e41782e2 in 1.0300 sec 2023-11-12T09:22:34,125 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-12T09:22:34,126 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-12T09:22:34,126 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-12T09:22:34,129 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-12T09:22:34,143 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-12T09:22:34,156 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-12T09:22:34,156 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/.hbase-snapshot/snapshot_backup_system 2023-11-12T09:22:34,159 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-12T09:22:34,159 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 65 2023-11-12T09:22:34,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.4800 sec 2023-11-12T09:22:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=65 2023-11-12T09:22:34,795 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(227): Deleting backup backup_1699780945813 ... 2023-11-12T09:22:34,796 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(437): Delete log files: .backup.manifest 2023-11-12T09:22:34,797 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(437): Delete log files: default 2023-11-12T09:22:34,798 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(445): Trying to cleanup up target dir : backup_1699780945813 2023-11-12T09:22:34,799 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(460): No data has been found in hdfs://localhost:37959/backupUT/backup_1699780945813/default/test-1699780901850. 2023-11-12T09:22:34,800 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.BackupUtils(467): hdfs://localhost:37959/backupUT/backup_1699780945813/default is empty, remove it. 2023-11-12T09:22:34,805 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-11-12T09:22:34,806 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-11-12T09:22:34,807 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupAdminImpl(276): Delete backup backup_1699780945813 completed. 2023-11-12T09:22:34,813 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1034): test-1699780901850 2023-11-12T09:22:34,816 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:22:34,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-11-12T09:22:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-12T09:22:34,826 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-12T09:22:34,826 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(657): Finish backup exclusive operation DELETE operation finished OK: backup_1699780945813 No failed backup MERGE operation found 2023-11-12T09:22:34,829 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-12T09:22:34,833 ERROR [Listener at jenkins-hbase3.apache.org/42237 {}] impl.BackupSystemTable(1579): Snapshot snapshot_backup_system does not exists 2023-11-12T09:22:34,833 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:34,833 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:34,858 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.ResourceChecker(175): after: backup.TestBackupDeleteWithFailures#testBackupDeleteWithFailures Thread=607 (was 584) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56022@0x52e1bf51 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1517) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1541) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$537/1554132319.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1c987ed5-shared-pool-22 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1313698232_16 at /127.0.0.1:49560 [Waiting for operation #24] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x358f80a6-shared-pool-26 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x358f80a6-shared-pool-28 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56022@0x52e1bf51-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: hconnection-0x1c987ed5-shared-pool-23 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,33463,1699780890205)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x358f80a6-shared-pool-25 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-958949052-172.31.12.81-1699780886949:blk_1073741905_1081, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x358f80a6-shared-pool-29 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x358f80a6-shared-pool-27 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1c987ed5-shared-pool-21 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56022@0x52e1bf51-SendThread(127.0.0.1:56022) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1394) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: hconnection-0x1c987ed5-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1457636924_16 at /127.0.0.1:49568 [Receiving block BP-958949052-172.31.12.81-1699780886949:blk_1073741905_1081] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1313698232_16 at /127.0.0.1:49604 [Waiting for operation #14] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,44111,1699780893376' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56022@0x296d92e1 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1517) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1541) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$537/1554132319.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/cluster_c9d24771-f798-4b7f-5e55-10c183a2c0d9/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1457636924_16 at /127.0.0.1:49570 [Receiving block BP-958949052-172.31.12.81-1699780886949:blk_1073741906_1082] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-958949052-172.31.12.81-1699780886949:blk_1073741906_1082, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,44111,1699780893376)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x358f80a6-shared-pool-30 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/cluster_c9d24771-f798-4b7f-5e55-10c183a2c0d9/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56022@0x296d92e1-SendThread(127.0.0.1:56022) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1394) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: hconnection-0x1c987ed5-shared-pool-25 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:56022@0x296d92e1-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1152330349_16 at /127.0.0.1:49686 [Waiting for operation #12] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=724 (was 689) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=169 (was 250), ProcessCount=172 (was 172), AvailableMemoryMB=6068 (was 5846) - AvailableMemoryMB LEAK? - 2023-11-12T09:22:34,858 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.ResourceChecker(130): Thread=607 is superior to 500 2023-11-12T09:22:34,866 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] master.MasterRpcServices(763): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1699780923644" 2023-11-12T09:22:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33463 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1699780923644 2023-11-12T09:22:34,872 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-11-12T09:22:34,872 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:34,872 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-11-12T09:22:34,872 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] util.JVMClusterUtil(257): Found active master hash=869660685, stopped=false 2023-11-12T09:22:34,873 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver 2023-11-12T09:22:34,873 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] master.ServerManager(896): Cluster shutdown requested of master=jenkins-hbase3.apache.org,33463,1699780890205 2023-11-12T09:22:34,877 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] procedure2.ProcedureExecutor(630): Stopping 2023-11-12T09:22:34,879 DEBUG [Listener at jenkins-hbase3.apache.org/42237 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:34,879 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,44111,1699780893376' ***** 2023-11-12T09:22:34,879 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-11-12T09:22:34,879 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-12T09:22:34,893 INFO [RS:0;jenkins-hbase3:44111 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@7d3ea42b{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-12T09:22:34,898 INFO [RS:0;jenkins-hbase3:44111 {}] server.AbstractConnector(383): Stopped ServerConnector@7c1f26d7{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:22:34,898 INFO [RS:0;jenkins-hbase3:44111 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-12T09:22:34,899 INFO [RS:0;jenkins-hbase3:44111 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3a055401{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-12T09:22:34,899 INFO [RS:0;jenkins-hbase3:44111 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2e744413{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/hadoop.log.dir/,STOPPED} 2023-11-12T09:22:34,901 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HeapMemoryManager(220): Stopping 2023-11-12T09:22:34,902 INFO [RS:0;jenkins-hbase3:44111 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-11-12T09:22:34,902 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-11-12T09:22:34,902 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-11-12T09:22:34,902 INFO [RS:0;jenkins-hbase3:44111 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-11-12T09:22:34,902 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(3528): Received CLOSE for 334467069bbeef2313578e4dfe9024f3 2023-11-12T09:22:34,902 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(3528): Received CLOSE for 0c6b0fe5f280595167fe10135a31e3d9 2023-11-12T09:22:34,902 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(3528): Received CLOSE for b74eabb10ae7ceb40fbd1cc11e47491c 2023-11-12T09:22:34,902 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(3528): Received CLOSE for 23ec3c63d01117206e0be8e22550ef70 2023-11-12T09:22:34,902 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(3528): Received CLOSE for fc49678786bc319888817a55d875958b 2023-11-12T09:22:34,902 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(3528): Received CLOSE for bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:34,902 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(3528): Received CLOSE for 9d3fa1e74299c566cf2ed2c7ca01ec35 2023-11-12T09:22:34,902 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,44111,1699780893376 2023-11-12T09:22:34,903 DEBUG [RS:0;jenkins-hbase3:44111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:34,903 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-11-12T09:22:34,903 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-11-12T09:22:34,903 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-11-12T09:22:34,903 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 334467069bbeef2313578e4dfe9024f3, disabling compactions & flushes 2023-11-12T09:22:34,903 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-11-12T09:22:34,903 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. 2023-11-12T09:22:34,903 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. 2023-11-12T09:22:34,903 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. after waiting 0 ms 2023-11-12T09:22:34,903 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. 2023-11-12T09:22:34,904 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-11-12T09:22:34,904 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1583): Online Regions={334467069bbeef2313578e4dfe9024f3=backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3., 0c6b0fe5f280595167fe10135a31e3d9=ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9., b74eabb10ae7ceb40fbd1cc11e47491c=ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c., 1588230740=hbase:meta,,1.1588230740, 23ec3c63d01117206e0be8e22550ef70=hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70., fc49678786bc319888817a55d875958b=ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b., bf8e1ef1f6635e037c32e6a8e41782e2=backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2., 9d3fa1e74299c566cf2ed2c7ca01ec35=test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35.} 2023-11-12T09:22:34,905 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-12T09:22:34,905 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-12T09:22:34,905 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-12T09:22:34,905 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-12T09:22:34,905 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-12T09:22:34,905 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=13.22 KB heapSize=23.11 KB 2023-11-12T09:22:34,905 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1609): Waiting on 0c6b0fe5f280595167fe10135a31e3d9, 1588230740, 23ec3c63d01117206e0be8e22550ef70, 334467069bbeef2313578e4dfe9024f3, 9d3fa1e74299c566cf2ed2c7ca01ec35, b74eabb10ae7ceb40fbd1cc11e47491c, bf8e1ef1f6635e037c32e6a8e41782e2, fc49678786bc319888817a55d875958b 2023-11-12T09:22:34,911 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system_bulk/334467069bbeef2313578e4dfe9024f3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:22:34,911 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:22:34,912 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. 2023-11-12T09:22:34,912 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 334467069bbeef2313578e4dfe9024f3: 2023-11-12T09:22:34,912 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1699780908912.334467069bbeef2313578e4dfe9024f3. 2023-11-12T09:22:34,913 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 0c6b0fe5f280595167fe10135a31e3d9, disabling compactions & flushes 2023-11-12T09:22:34,913 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. 2023-11-12T09:22:34,913 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. 2023-11-12T09:22:34,913 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. after waiting 0 ms 2023-11-12T09:22:34,913 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. 2023-11-12T09:22:34,917 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns4/test-16997809018503/0c6b0fe5f280595167fe10135a31e3d9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:22:34,918 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:22:34,920 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. 2023-11-12T09:22:34,920 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 0c6b0fe5f280595167fe10135a31e3d9: 2023-11-12T09:22:34,920 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16997809018503,,1699780906438.0c6b0fe5f280595167fe10135a31e3d9. 2023-11-12T09:22:34,921 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b74eabb10ae7ceb40fbd1cc11e47491c, disabling compactions & flushes 2023-11-12T09:22:34,921 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. 2023-11-12T09:22:34,921 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. 2023-11-12T09:22:34,921 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. after waiting 0 ms 2023-11-12T09:22:34,921 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. 2023-11-12T09:22:34,928 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns3/test-16997809018502/b74eabb10ae7ceb40fbd1cc11e47491c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-12T09:22:34,928 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:22:34,929 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. 2023-11-12T09:22:34,929 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b74eabb10ae7ceb40fbd1cc11e47491c: 2023-11-12T09:22:34,929 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16997809018502,,1699780905815.b74eabb10ae7ceb40fbd1cc11e47491c. 2023-11-12T09:22:34,930 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 23ec3c63d01117206e0be8e22550ef70, disabling compactions & flushes 2023-11-12T09:22:34,930 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. 2023-11-12T09:22:34,930 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. 2023-11-12T09:22:34,930 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. after waiting 0 ms 2023-11-12T09:22:34,930 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. 2023-11-12T09:22:34,930 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 23ec3c63d01117206e0be8e22550ef70 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-11-12T09:22:34,937 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740/.tmp/info/591fab3e049a417d8a587faa73bd8a88 is 159, key is ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b./info:regioninfo/1699780904803/Put/seqid=0 2023-11-12T09:22:34,944 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.18 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740/.tmp/info/591fab3e049a417d8a587faa73bd8a88 2023-11-12T09:22:34,948 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/namespace/23ec3c63d01117206e0be8e22550ef70/.tmp/info/8304744b063446a58894d3b322ee70db is 45, key is default/info:d/1699780897428/Put/seqid=0 2023-11-12T09:22:34,965 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740/.tmp/table/b5703e758f9b43099c5e90da7fc9d147 is 59, key is ns2:test-16997809018501/table:state/1699780904813/Put/seqid=0 2023-11-12T09:22:34,979 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-11-12T09:22:35,024 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.23076923076923078, tune throughput to 61.54 MB/second 2023-11-12T09:22:35,024 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-11-12T09:22:35,024 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-11-12T09:22:35,105 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 23ec3c63d01117206e0be8e22550ef70, 9d3fa1e74299c566cf2ed2c7ca01ec35, bf8e1ef1f6635e037c32e6a8e41782e2, fc49678786bc319888817a55d875958b 2023-11-12T09:22:35,306 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 23ec3c63d01117206e0be8e22550ef70, 9d3fa1e74299c566cf2ed2c7ca01ec35, bf8e1ef1f6635e037c32e6a8e41782e2, fc49678786bc319888817a55d875958b 2023-11-12T09:22:35,361 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/namespace/23ec3c63d01117206e0be8e22550ef70/.tmp/info/8304744b063446a58894d3b322ee70db 2023-11-12T09:22:35,368 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/namespace/23ec3c63d01117206e0be8e22550ef70/.tmp/info/8304744b063446a58894d3b322ee70db as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/namespace/23ec3c63d01117206e0be8e22550ef70/info/8304744b063446a58894d3b322ee70db 2023-11-12T09:22:35,370 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740/.tmp/table/b5703e758f9b43099c5e90da7fc9d147 2023-11-12T09:22:35,374 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/namespace/23ec3c63d01117206e0be8e22550ef70/info/8304744b063446a58894d3b322ee70db, entries=7, sequenceid=11, filesize=5.1 K 2023-11-12T09:22:35,375 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 23ec3c63d01117206e0be8e22550ef70 in 445ms, sequenceid=11, compaction requested=false 2023-11-12T09:22:35,378 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740/.tmp/info/591fab3e049a417d8a587faa73bd8a88 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740/info/591fab3e049a417d8a587faa73bd8a88 2023-11-12T09:22:35,384 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/namespace/23ec3c63d01117206e0be8e22550ef70/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-11-12T09:22:35,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:22:35,386 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. 2023-11-12T09:22:35,386 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 23ec3c63d01117206e0be8e22550ef70: 2023-11-12T09:22:35,386 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1699780896035.23ec3c63d01117206e0be8e22550ef70. 2023-11-12T09:22:35,386 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing fc49678786bc319888817a55d875958b, disabling compactions & flushes 2023-11-12T09:22:35,386 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. 2023-11-12T09:22:35,386 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. 2023-11-12T09:22:35,386 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. after waiting 0 ms 2023-11-12T09:22:35,386 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. 2023-11-12T09:22:35,387 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing fc49678786bc319888817a55d875958b 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-12T09:22:35,388 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740/info/591fab3e049a417d8a587faa73bd8a88, entries=72, sequenceid=59, filesize=14.3 K 2023-11-12T09:22:35,389 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740/.tmp/table/b5703e758f9b43099c5e90da7fc9d147 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740/table/b5703e758f9b43099c5e90da7fc9d147 2023-11-12T09:22:35,396 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740/table/b5703e758f9b43099c5e90da7fc9d147, entries=15, sequenceid=59, filesize=5.8 K 2023-11-12T09:22:35,397 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~13.22 KB/13542, heapSize ~22.83 KB/23376, currentSize=0 B/0 for 1588230740 in 492ms, sequenceid=59, compaction requested=false 2023-11-12T09:22:35,408 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/hbase/meta/1588230740/recovered.edits/62.seqid, newMaxSeqId=62, maxSeqId=1 2023-11-12T09:22:35,409 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:22:35,409 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-12T09:22:35,409 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-12T09:22:35,409 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-12T09:22:35,409 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-11-12T09:22:35,413 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns2/test-16997809018501/fc49678786bc319888817a55d875958b/.tmp/f/7987b01929794a518100622a30dad3ae is 37, key is row10/f:q1/1699780905729/Put/seqid=0 2023-11-12T09:22:35,506 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1609): Waiting on 9d3fa1e74299c566cf2ed2c7ca01ec35, bf8e1ef1f6635e037c32e6a8e41782e2, fc49678786bc319888817a55d875958b 2023-11-12T09:22:35,706 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1609): Waiting on 9d3fa1e74299c566cf2ed2c7ca01ec35, bf8e1ef1f6635e037c32e6a8e41782e2, fc49678786bc319888817a55d875958b 2023-11-12T09:22:35,818 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns2/test-16997809018501/fc49678786bc319888817a55d875958b/.tmp/f/7987b01929794a518100622a30dad3ae 2023-11-12T09:22:35,825 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns2/test-16997809018501/fc49678786bc319888817a55d875958b/.tmp/f/7987b01929794a518100622a30dad3ae as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns2/test-16997809018501/fc49678786bc319888817a55d875958b/f/7987b01929794a518100622a30dad3ae 2023-11-12T09:22:35,830 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns2/test-16997809018501/fc49678786bc319888817a55d875958b/f/7987b01929794a518100622a30dad3ae, entries=99, sequenceid=103, filesize=8.2 K 2023-11-12T09:22:35,831 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for fc49678786bc319888817a55d875958b in 445ms, sequenceid=103, compaction requested=false 2023-11-12T09:22:35,838 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/ns2/test-16997809018501/fc49678786bc319888817a55d875958b/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-11-12T09:22:35,839 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:22:35,840 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. 2023-11-12T09:22:35,840 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for fc49678786bc319888817a55d875958b: 2023-11-12T09:22:35,840 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16997809018501,,1699780903597.fc49678786bc319888817a55d875958b. 2023-11-12T09:22:35,840 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing bf8e1ef1f6635e037c32e6a8e41782e2, disabling compactions & flushes 2023-11-12T09:22:35,840 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:35,841 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:35,841 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. after waiting 0 ms 2023-11-12T09:22:35,841 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:35,841 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing bf8e1ef1f6635e037c32e6a8e41782e2 2/2 column families, dataSize=223 B heapSize=1.01 KB 2023-11-12T09:22:35,845 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/84f4a3d73a3b485d8c10221fd2540e91 is 91, key is incrbackupset:hdfs://localhost:37959/backupUT/meta:test-1699780901850/1699780954814/Put/seqid=0 2023-11-12T09:22:35,906 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-11-12T09:22:35,907 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1583): Online Regions={bf8e1ef1f6635e037c32e6a8e41782e2=backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2., 9d3fa1e74299c566cf2ed2c7ca01ec35=test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35.} 2023-11-12T09:22:35,907 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1609): Waiting on 9d3fa1e74299c566cf2ed2c7ca01ec35, bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:36,021 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 23ec3c63d01117206e0be8e22550ef70 changed from -1.0 to 0.0, refreshing cache 2023-11-12T09:22:36,107 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1609): Waiting on 9d3fa1e74299c566cf2ed2c7ca01ec35, bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:36,252 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/84f4a3d73a3b485d8c10221fd2540e91 2023-11-12T09:22:36,259 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 84f4a3d73a3b485d8c10221fd2540e91 2023-11-12T09:22:36,262 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/ecc0487009544cdf813e0ca39cd3e761 is 59, key is session:backup_1699780945813/session:/1699780954806/DeleteFamily/seqid=0 2023-11-12T09:22:36,268 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/ecc0487009544cdf813e0ca39cd3e761 2023-11-12T09:22:36,274 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ecc0487009544cdf813e0ca39cd3e761 2023-11-12T09:22:36,276 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/meta/84f4a3d73a3b485d8c10221fd2540e91 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/84f4a3d73a3b485d8c10221fd2540e91 2023-11-12T09:22:36,287 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 84f4a3d73a3b485d8c10221fd2540e91 2023-11-12T09:22:36,287 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/meta/84f4a3d73a3b485d8c10221fd2540e91, entries=2, sequenceid=87, filesize=5.2 K 2023-11-12T09:22:36,288 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/.tmp/session/ecc0487009544cdf813e0ca39cd3e761 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/ecc0487009544cdf813e0ca39cd3e761 2023-11-12T09:22:36,294 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ecc0487009544cdf813e0ca39cd3e761 2023-11-12T09:22:36,294 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/session/ecc0487009544cdf813e0ca39cd3e761, entries=2, sequenceid=87, filesize=5.2 K 2023-11-12T09:22:36,295 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~223 B/223, heapSize ~1000 B/1000, currentSize=0 B/0 for bf8e1ef1f6635e037c32e6a8e41782e2 in 454ms, sequenceid=87, compaction requested=true 2023-11-12T09:22:36,309 DEBUG [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1609): Waiting on 9d3fa1e74299c566cf2ed2c7ca01ec35, bf8e1ef1f6635e037c32e6a8e41782e2 2023-11-12T09:22:36,320 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/backup/system/bf8e1ef1f6635e037c32e6a8e41782e2/recovered.edits/90.seqid, newMaxSeqId=90, maxSeqId=34 2023-11-12T09:22:36,321 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:22:36,321 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:36,321 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for bf8e1ef1f6635e037c32e6a8e41782e2: 2023-11-12T09:22:36,322 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1699780907781.bf8e1ef1f6635e037c32e6a8e41782e2. 2023-11-12T09:22:36,322 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9d3fa1e74299c566cf2ed2c7ca01ec35, disabling compactions & flushes 2023-11-12T09:22:36,322 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:22:36,322 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:22:36,322 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. after waiting 0 ms 2023-11-12T09:22:36,322 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:22:36,326 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/data/default/test-1699780901850/9d3fa1e74299c566cf2ed2c7ca01ec35/recovered.edits/108.seqid, newMaxSeqId=108, maxSeqId=1 2023-11-12T09:22:36,326 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-12T09:22:36,327 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:22:36,327 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9d3fa1e74299c566cf2ed2c7ca01ec35: 2023-11-12T09:22:36,327 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1699780901850,,1699780902343.9d3fa1e74299c566cf2ed2c7ca01ec35. 2023-11-12T09:22:36,510 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,44111,1699780893376; all regions closed. 2023-11-12T09:22:36,519 DEBUG [RS:0;jenkins-hbase3:44111 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/oldWALs 2023-11-12T09:22:36,519 INFO [RS:0;jenkins-hbase3:44111 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C44111%2C1699780893376.meta:.meta(num 1699780947981) 2023-11-12T09:22:36,525 DEBUG [RS:0;jenkins-hbase3:44111 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/oldWALs 2023-11-12T09:22:36,525 INFO [RS:0;jenkins-hbase3:44111 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C44111%2C1699780893376:(num 1699780948000) 2023-11-12T09:22:36,525 DEBUG [RS:0;jenkins-hbase3:44111 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:36,525 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.LeaseManager(133): Closed leases 2023-11-12T09:22:36,525 INFO [RS:0;jenkins-hbase3:44111 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-11-12T09:22:36,526 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-12T09:22:36,527 INFO [RS:0;jenkins-hbase3:44111 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:44111 2023-11-12T09:22:36,536 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,44111,1699780893376] 2023-11-12T09:22:36,536 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,44111,1699780893376; numProcessing=1 2023-11-12T09:22:36,537 INFO [RegionServerTracker-0 {}] master.ServerManager(561): Cluster shutdown set; jenkins-hbase3.apache.org,44111,1699780893376 expired; onlineServers=0 2023-11-12T09:22:36,537 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,33463,1699780890205' ***** 2023-11-12T09:22:36,537 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-11-12T09:22:36,538 DEBUG [M:0;jenkins-hbase3:33463 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cde0893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-11-12T09:22:36,538 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-12T09:22:36,580 INFO [M:0;jenkins-hbase3:33463 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@36f4baa6{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-12T09:22:36,580 INFO [M:0;jenkins-hbase3:33463 {}] server.AbstractConnector(383): Stopped ServerConnector@72c7f512{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:22:36,580 INFO [M:0;jenkins-hbase3:33463 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-12T09:22:36,581 INFO [M:0;jenkins-hbase3:33463 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5a55de6b{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-12T09:22:36,581 INFO [M:0;jenkins-hbase3:33463 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@58a5effe{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/hadoop.log.dir/,STOPPED} 2023-11-12T09:22:36,582 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,33463,1699780890205 2023-11-12T09:22:36,582 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,33463,1699780890205; all regions closed. 2023-11-12T09:22:36,582 DEBUG [M:0;jenkins-hbase3:33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:36,582 INFO [M:0;jenkins-hbase3:33463 {}] master.HMaster(1666): Stopping master jetty server 2023-11-12T09:22:36,582 INFO [M:0;jenkins-hbase3:33463 {}] server.AbstractConnector(383): Stopped ServerConnector@6fae22d8{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-12T09:22:36,583 DEBUG [M:0;jenkins-hbase3:33463 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-11-12T09:22:36,583 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-11-12T09:22:36,583 INFO [M:0;jenkins-hbase3:33463 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-11-12T09:22:36,583 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699780894763 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1699780894763,5,FailOnTimeoutGroup] 2023-11-12T09:22:36,583 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699780894763 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1699780894763,5,FailOnTimeoutGroup] 2023-11-12T09:22:36,583 DEBUG [M:0;jenkins-hbase3:33463 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-12T09:22:36,583 DEBUG [M:0;jenkins-hbase3:33463 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-11-12T09:22:36,584 INFO [M:0;jenkins-hbase3:33463 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-11-12T09:22:36,584 DEBUG [M:0;jenkins-hbase3:33463 {}] master.HMaster(1689): Stopping service threads 2023-11-12T09:22:36,584 INFO [M:0;jenkins-hbase3:33463 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-11-12T09:22:36,584 ERROR [M:0;jenkins-hbase3:33463 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-11-12T09:22:36,585 INFO [M:0;jenkins-hbase3:33463 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-11-12T09:22:36,585 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-11-12T09:22:36,585 WARN [M:0;jenkins-hbase3:33463 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-11-12T09:22:36,585 INFO [M:0;jenkins-hbase3:33463 {}] assignment.AssignmentManager(383): Stopping assignment manager 2023-11-12T09:22:36,585 INFO [M:0;jenkins-hbase3:33463 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-11-12T09:22:36,586 DEBUG [M:0;jenkins-hbase3:33463 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-12T09:22:36,598 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:22:36,598 DEBUG [M:0;jenkins-hbase3:33463 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:22:36,598 DEBUG [M:0;jenkins-hbase3:33463 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-12T09:22:36,598 DEBUG [M:0;jenkins-hbase3:33463 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:22:36,598 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=254.52 KB heapSize=306.29 KB 2023-11-12T09:22:36,610 DEBUG [M:0;jenkins-hbase3:33463 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70fbc8d199ab44eaaa723eabb85405ea is 88, key is hbase:meta,,1/info:sn/1699780895362/Put/seqid=0 2023-11-12T09:22:36,636 INFO [RS:0;jenkins-hbase3:44111 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,44111,1699780893376; zookeeper connection closed. 2023-11-12T09:22:36,636 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@535cac9c {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@535cac9c 2023-11-12T09:22:36,637 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-11-12T09:22:37,015 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70fbc8d199ab44eaaa723eabb85405ea 2023-11-12T09:22:37,037 DEBUG [M:0;jenkins-hbase3:33463 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c13501f93125468bb8084320b83c8f26 is 2093, key is \x00\x00\x00\x00\x00\x00\x00@/proc:d/1699780952522/Put/seqid=0 2023-11-12T09:22:37,442 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=253.92 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c13501f93125468bb8084320b83c8f26 2023-11-12T09:22:37,449 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c13501f93125468bb8084320b83c8f26 2023-11-12T09:22:37,464 DEBUG [M:0;jenkins-hbase3:33463 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/594feb3c881c434d9a8e700c1e1d3923 is 82, key is jenkins-hbase3.apache.org,44111,1699780893376/rs:state/1699780894870/Put/seqid=0 2023-11-12T09:22:37,470 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/594feb3c881c434d9a8e700c1e1d3923 2023-11-12T09:22:37,476 DEBUG [M:0;jenkins-hbase3:33463 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70fbc8d199ab44eaaa723eabb85405ea as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/70fbc8d199ab44eaaa723eabb85405ea 2023-11-12T09:22:37,481 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/70fbc8d199ab44eaaa723eabb85405ea, entries=8, sequenceid=617, filesize=5.5 K 2023-11-12T09:22:37,482 DEBUG [M:0;jenkins-hbase3:33463 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c13501f93125468bb8084320b83c8f26 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c13501f93125468bb8084320b83c8f26 2023-11-12T09:22:37,487 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c13501f93125468bb8084320b83c8f26 2023-11-12T09:22:37,487 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c13501f93125468bb8084320b83c8f26, entries=66, sequenceid=617, filesize=22.2 K 2023-11-12T09:22:37,488 DEBUG [M:0;jenkins-hbase3:33463 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/594feb3c881c434d9a8e700c1e1d3923 as hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/594feb3c881c434d9a8e700c1e1d3923 2023-11-12T09:22:37,493 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37959/user/jenkins/test-data/6903fd7a-8b92-2b5a-c2cf-f40421af8dfe/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/594feb3c881c434d9a8e700c1e1d3923, entries=1, sequenceid=617, filesize=5.1 K 2023-11-12T09:22:37,494 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.HRegion(3022): Finished flush of dataSize ~254.52 KB/260625, heapSize ~305.99 KB/313336, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 896ms, sequenceid=617, compaction requested=false 2023-11-12T09:22:37,498 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-12T09:22:37,499 DEBUG [M:0;jenkins-hbase3:33463 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-12T09:22:37,502 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-12T09:22:37,502 INFO [M:0;jenkins-hbase3:33463 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-11-12T09:22:37,502 INFO [M:0;jenkins-hbase3:33463 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-11-12T09:22:37,503 INFO [M:0;jenkins-hbase3:33463 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:33463 2023-11-12T09:22:37,607 INFO [M:0;jenkins-hbase3:33463 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,33463,1699780890205; zookeeper connection closed. 2023-11-12T09:22:37,610 WARN [Listener at jenkins-hbase3.apache.org/42237 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-12T09:22:37,615 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:22:37,724 WARN [BP-958949052-172.31.12.81-1699780886949 heartbeating to localhost/127.0.0.1:37959 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-12T09:22:37,725 WARN [BP-958949052-172.31.12.81-1699780886949 heartbeating to localhost/127.0.0.1:37959 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-958949052-172.31.12.81-1699780886949 (Datanode Uuid 41ceac43-a147-4903-993b-0fa32b96298f) service to localhost/127.0.0.1:37959 2023-11-12T09:22:37,726 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/cluster_c9d24771-f798-4b7f-5e55-10c183a2c0d9/dfs/data/data1/current/BP-958949052-172.31.12.81-1699780886949 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:22:37,727 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/97b61604-57c1-98c4-4cfc-1bc2f19d1ee2/cluster_c9d24771-f798-4b7f-5e55-10c183a2c0d9/dfs/data/data2/current/BP-958949052-172.31.12.81-1699780886949 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-12T09:22:37,782 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-12T09:22:37,942 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-11-12T09:22:37,942 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-11-12T09:22:37,953 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:22:38,078 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:22:38,204 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:22:38,204 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:22:38,312 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-11-12T09:22:38,325 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-11-12T09:22:38,326 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:22:38,333 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-11-12T09:22:38,436 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-12T09:22:38,436 INFO [Listener at jenkins-hbase3.apache.org/42237 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped