2023-11-19 01:28:45,295 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1d296da 2023-11-19 01:28:45,312 main DEBUG Took 0.013763 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-11-19 01:28:45,312 main DEBUG PluginManager 'Core' found 129 plugins 2023-11-19 01:28:45,312 main DEBUG PluginManager 'Level' found 0 plugins 2023-11-19 01:28:45,313 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-11-19 01:28:45,315 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:28:45,327 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-11-19 01:28:45,345 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,347 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:28:45,347 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,347 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:28:45,348 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,348 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:28:45,349 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,349 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:28:45,349 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,350 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:28:45,351 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,351 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:28:45,351 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,352 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:28:45,352 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,353 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:28:45,353 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,353 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:28:45,354 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,354 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:28:45,355 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,355 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:28:45,356 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,356 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:28:45,356 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,357 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-11-19 01:28:45,359 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:28:45,361 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-11-19 01:28:45,363 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-11-19 01:28:45,363 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-11-19 01:28:45,365 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-11-19 01:28:45,365 main DEBUG PluginManager 'Converter' found 47 plugins 2023-11-19 01:28:45,381 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-11-19 01:28:45,384 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-11-19 01:28:45,385 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-11-19 01:28:45,386 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-11-19 01:28:45,386 main DEBUG createAppenders(={Console}) 2023-11-19 01:28:45,387 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1d296da initialized 2023-11-19 01:28:45,388 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1d296da 2023-11-19 01:28:45,388 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1d296da OK. 2023-11-19 01:28:45,389 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-11-19 01:28:45,389 main DEBUG OutputStream closed 2023-11-19 01:28:45,390 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-11-19 01:28:45,390 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-11-19 01:28:45,391 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@39529185 OK 2023-11-19 01:28:45,444 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-11-19 01:28:45,446 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-11-19 01:28:45,447 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-11-19 01:28:45,449 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-11-19 01:28:45,449 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-11-19 01:28:45,450 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-11-19 01:28:45,450 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-11-19 01:28:45,451 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-11-19 01:28:45,451 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-11-19 01:28:45,452 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-11-19 01:28:45,452 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-11-19 01:28:45,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-11-19 01:28:45,453 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-11-19 01:28:45,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-11-19 01:28:45,454 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-11-19 01:28:45,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-11-19 01:28:45,455 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-11-19 01:28:45,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-11-19 01:28:45,463 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-19 01:28:45,463 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6a84a97d) with optional ClassLoader: null 2023-11-19 01:28:45,464 main DEBUG Shutdown hook enabled. Registering a new one. 2023-11-19 01:28:45,465 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@6a84a97d] started OK. 2023-11-19T01:28:45,499 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures timeout: 13 mins 2023-11-19 01:28:45,504 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-11-19 01:28:45,504 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-19T01:28:45,811 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012 2023-11-19T01:28:45,816 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-19T01:28:45,818 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:28:45,819 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-11-19T01:28:45,823 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/cluster_e2bb288f-7ce7-cac6-514c-081c116c782a, deleteOnExit=true 2023-11-19T01:28:45,823 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-11-19T01:28:45,824 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/test.cache.data in system properties and HBase conf 2023-11-19T01:28:45,824 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop.tmp.dir in system properties and HBase conf 2023-11-19T01:28:45,824 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop.log.dir in system properties and HBase conf 2023-11-19T01:28:45,825 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-19T01:28:45,825 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-19T01:28:45,825 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-19T01:28:45,930 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-11-19T01:28:46,431 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-11-19T01:28:46,438 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:28:46,439 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:28:46,439 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-19T01:28:46,440 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:28:46,440 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-19T01:28:46,441 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-19T01:28:46,441 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:28:46,442 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:28:46,442 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-19T01:28:46,443 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/nfs.dump.dir in system properties and HBase conf 2023-11-19T01:28:46,443 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/java.io.tmpdir in system properties and HBase conf 2023-11-19T01:28:46,443 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:28:46,444 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-19T01:28:46,444 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-11-19T01:28:46,941 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-19T01:28:46,946 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-19T01:28:47,231 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-11-19T01:28:47,367 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@45e86cbb via org.mortbay.log.Slf4jLog 2023-11-19T01:28:47,382 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:28:47,421 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:28:47,457 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/java.io.tmpdir/Jetty_localhost_37867_hdfs____o1n2db/webapp 2023-11-19T01:28:47,591 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37867 2023-11-19T01:28:47,615 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-19T01:28:47,615 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-19T01:28:48,161 WARN [Listener at localhost/39587 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:28:48,223 WARN [Listener at localhost/39587 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-19T01:28:48,242 WARN [Listener at localhost/39587 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:28:48,248 INFO [Listener at localhost/39587 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:28:48,253 INFO [Listener at localhost/39587 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/java.io.tmpdir/Jetty_localhost_41713_datanode____e8ozj7/webapp 2023-11-19T01:28:48,357 INFO [Listener at localhost/39587 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:41713 2023-11-19T01:28:48,650 WARN [Listener at localhost/35467 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:28:48,922 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-19T01:28:49,093 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x2491bd2b4024a7ce: Processing first storage report for DS-411aec13-6b5e-41f2-9713-9adb5ce871b9 from datanode fe2559e6-db39-4175-a0c5-755883551972 2023-11-19T01:28:49,095 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x2491bd2b4024a7ce: from storage DS-411aec13-6b5e-41f2-9713-9adb5ce871b9 node DatanodeRegistration(127.0.0.1:41437, datanodeUuid=fe2559e6-db39-4175-a0c5-755883551972, infoPort=41869, infoSecurePort=0, ipcPort=35467, storageInfo=lv=-57;cid=testClusterID;nsid=1871652494;c=1700357327020), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-11-19T01:28:49,095 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x2491bd2b4024a7ce: Processing first storage report for DS-dc668251-e5bd-43db-b0b8-cd51b19cc694 from datanode fe2559e6-db39-4175-a0c5-755883551972 2023-11-19T01:28:49,095 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x2491bd2b4024a7ce: from storage DS-dc668251-e5bd-43db-b0b8-cd51b19cc694 node DatanodeRegistration(127.0.0.1:41437, datanodeUuid=fe2559e6-db39-4175-a0c5-755883551972, infoPort=41869, infoSecurePort=0, ipcPort=35467, storageInfo=lv=-57;cid=testClusterID;nsid=1871652494;c=1700357327020), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2023-11-19T01:28:49,165 DEBUG [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012 2023-11-19T01:28:49,308 INFO [Listener at localhost/35467 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/cluster_e2bb288f-7ce7-cac6-514c-081c116c782a/zookeeper_0, clientPort=59682, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/cluster_e2bb288f-7ce7-cac6-514c-081c116c782a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/cluster_e2bb288f-7ce7-cac6-514c-081c116c782a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2023-11-19T01:28:49,326 INFO [Listener at localhost/35467 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=59682 2023-11-19T01:28:49,334 INFO [Listener at localhost/35467 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:28:49,336 INFO [Listener at localhost/35467 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:28:49,990 INFO [Listener at localhost/35467 {}] util.FSUtils(462): Created version file at hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa with version=8 2023-11-19T01:28:49,991 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/hbase-staging 2023-11-19T01:28:50,007 DEBUG [Listener at localhost/35467 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-11-19T01:28:50,007 DEBUG [Listener at localhost/35467 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-11-19T01:28:50,007 DEBUG [Listener at localhost/35467 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-11-19T01:28:50,007 DEBUG [Listener at localhost/35467 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-11-19T01:28:50,246 DEBUG [Listener at localhost/35467 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-11-19T01:28:51,013 INFO [Listener at localhost/35467 {}] client.ConnectionUtils(128): master/jenkins-hbase6:0 server-side Connection retries=45 2023-11-19T01:28:51,061 INFO [Listener at localhost/35467 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:28:51,061 INFO [Listener at localhost/35467 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-19T01:28:51,062 INFO [Listener at localhost/35467 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-19T01:28:51,062 INFO [Listener at localhost/35467 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:28:51,062 INFO [Listener at localhost/35467 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-19T01:28:51,215 INFO [Listener at localhost/35467 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-19T01:28:51,346 INFO [Listener at localhost/35467 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-11-19T01:28:51,365 DEBUG [Listener at localhost/35467 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-11-19T01:28:51,371 INFO [Listener at localhost/35467 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-19T01:28:51,419 DEBUG [Listener at localhost/35467 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 11135 (auto-detected) 2023-11-19T01:28:51,420 DEBUG [Listener at localhost/35467 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:55:ff:fe:37:d3:4d (auto-detected) 2023-11-19T01:28:51,457 INFO [Listener at localhost/35467 {}] ipc.NettyRpcServer(192): Bind to /172.31.2.20:36603 2023-11-19T01:28:51,471 INFO [Listener at localhost/35467 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:28:51,473 INFO [Listener at localhost/35467 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:28:51,495 INFO [Listener at localhost/35467 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:36603 connecting to ZooKeeper ensemble=127.0.0.1:59682 2023-11-19T01:28:51,546 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:366030x0, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-19T01:28:51,549 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:36603-0x10123a5ddbf0000 connected 2023-11-19T01:28:51,583 DEBUG [Listener at localhost/35467 {}] zookeeper.ZKUtil(113): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-19T01:28:51,586 DEBUG [Listener at localhost/35467 {}] zookeeper.ZKUtil(113): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-19T01:28:51,589 DEBUG [Listener at localhost/35467 {}] zookeeper.ZKUtil(113): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-19T01:28:51,600 DEBUG [Listener at localhost/35467 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36603 2023-11-19T01:28:51,600 DEBUG [Listener at localhost/35467 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36603 2023-11-19T01:28:51,601 DEBUG [Listener at localhost/35467 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36603 2023-11-19T01:28:51,601 DEBUG [Listener at localhost/35467 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36603 2023-11-19T01:28:51,601 DEBUG [Listener at localhost/35467 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36603 2023-11-19T01:28:51,656 INFO [Listener at localhost/35467 {}] log.Log(170): Logging initialized @7041ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-11-19T01:28:51,785 INFO [Listener at localhost/35467 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-19T01:28:51,786 INFO [Listener at localhost/35467 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-19T01:28:51,787 INFO [Listener at localhost/35467 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-19T01:28:51,789 INFO [Listener at localhost/35467 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-11-19T01:28:51,789 INFO [Listener at localhost/35467 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-19T01:28:51,789 INFO [Listener at localhost/35467 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-19T01:28:51,793 INFO [Listener at localhost/35467 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-19T01:28:51,929 INFO [Listener at localhost/35467 {}] http.HttpServer(1227): Jetty bound to port 36005 2023-11-19T01:28:51,931 INFO [Listener at localhost/35467 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:28:51,967 INFO [Listener at localhost/35467 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:28:51,971 INFO [Listener at localhost/35467 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@50df4d49{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop.log.dir/,AVAILABLE} 2023-11-19T01:28:51,971 INFO [Listener at localhost/35467 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:28:51,971 INFO [Listener at localhost/35467 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@68d66e26{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-19T01:28:52,034 INFO [Listener at localhost/35467 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-19T01:28:52,048 INFO [Listener at localhost/35467 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-19T01:28:52,048 INFO [Listener at localhost/35467 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-19T01:28:52,050 INFO [Listener at localhost/35467 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-11-19T01:28:52,058 INFO [Listener at localhost/35467 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:28:53,198 INFO [Listener at localhost/35467 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@52673c48{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-19T01:28:53,209 INFO [Listener at localhost/35467 {}] server.AbstractConnector(333): Started ServerConnector@48dc6477{HTTP/1.1, (http/1.1)}{0.0.0.0:36005} 2023-11-19T01:28:53,210 INFO [Listener at localhost/35467 {}] server.Server(415): Started @8595ms 2023-11-19T01:28:53,213 INFO [Listener at localhost/35467 {}] master.HMaster(486): hbase.rootdir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa, hbase.cluster.distributed=false 2023-11-19T01:28:53,274 INFO [Listener at localhost/35467 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase6:0 server-side Connection retries=45 2023-11-19T01:28:53,275 INFO [Listener at localhost/35467 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:28:53,275 INFO [Listener at localhost/35467 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-19T01:28:53,275 INFO [Listener at localhost/35467 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-19T01:28:53,275 INFO [Listener at localhost/35467 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:28:53,275 INFO [Listener at localhost/35467 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-19T01:28:53,284 INFO [Listener at localhost/35467 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-19T01:28:53,286 INFO [Listener at localhost/35467 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-19T01:28:53,287 INFO [Listener at localhost/35467 {}] ipc.NettyRpcServer(192): Bind to /172.31.2.20:40967 2023-11-19T01:28:53,289 INFO [Listener at localhost/35467 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-11-19T01:28:53,297 DEBUG [Listener at localhost/35467 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-11-19T01:28:53,297 INFO [Listener at localhost/35467 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:28:53,300 INFO [Listener at localhost/35467 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:28:53,303 INFO [Listener at localhost/35467 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:40967 connecting to ZooKeeper ensemble=127.0.0.1:59682 2023-11-19T01:28:53,306 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:409670x0, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-19T01:28:53,307 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:40967-0x10123a5ddbf0001 connected 2023-11-19T01:28:53,307 DEBUG [Listener at localhost/35467 {}] zookeeper.ZKUtil(113): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-19T01:28:53,309 DEBUG [Listener at localhost/35467 {}] zookeeper.ZKUtil(113): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-19T01:28:53,310 DEBUG [Listener at localhost/35467 {}] zookeeper.ZKUtil(113): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-19T01:28:53,312 DEBUG [Listener at localhost/35467 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40967 2023-11-19T01:28:53,312 DEBUG [Listener at localhost/35467 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40967 2023-11-19T01:28:53,313 DEBUG [Listener at localhost/35467 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40967 2023-11-19T01:28:53,313 DEBUG [Listener at localhost/35467 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40967 2023-11-19T01:28:53,313 DEBUG [Listener at localhost/35467 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40967 2023-11-19T01:28:53,317 INFO [Listener at localhost/35467 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-19T01:28:53,317 INFO [Listener at localhost/35467 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-19T01:28:53,317 INFO [Listener at localhost/35467 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-19T01:28:53,318 INFO [Listener at localhost/35467 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-11-19T01:28:53,318 INFO [Listener at localhost/35467 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-19T01:28:53,318 INFO [Listener at localhost/35467 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-19T01:28:53,319 INFO [Listener at localhost/35467 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-19T01:28:53,320 INFO [Listener at localhost/35467 {}] http.HttpServer(1227): Jetty bound to port 41801 2023-11-19T01:28:53,321 INFO [Listener at localhost/35467 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:28:53,322 INFO [Listener at localhost/35467 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:28:53,322 INFO [Listener at localhost/35467 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3040caf3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop.log.dir/,AVAILABLE} 2023-11-19T01:28:53,323 INFO [Listener at localhost/35467 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:28:53,323 INFO [Listener at localhost/35467 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1355a3b7{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-19T01:28:53,335 INFO [Listener at localhost/35467 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-19T01:28:53,336 INFO [Listener at localhost/35467 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-19T01:28:53,336 INFO [Listener at localhost/35467 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-19T01:28:53,336 INFO [Listener at localhost/35467 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-11-19T01:28:53,337 INFO [Listener at localhost/35467 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:28:53,342 INFO [Listener at localhost/35467 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@110ad48e{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-19T01:28:53,342 INFO [Listener at localhost/35467 {}] server.AbstractConnector(333): Started ServerConnector@4dfe1c86{HTTP/1.1, (http/1.1)}{0.0.0.0:41801} 2023-11-19T01:28:53,343 INFO [Listener at localhost/35467 {}] server.Server(415): Started @8728ms 2023-11-19T01:28:53,347 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:28:53,350 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@1ea10a7a{HTTP/1.1, (http/1.1)}{0.0.0.0:38437} 2023-11-19T01:28:53,350 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.Server(415): Started @8735ms 2023-11-19T01:28:53,350 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(2389): Adding backup master ZNode /1/backup-masters/jenkins-hbase6.apache.org,36603,1700357330230 2023-11-19T01:28:53,363 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:28:53,363 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:28:53,366 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase6.apache.org,36603,1700357330230 2023-11-19T01:28:53,370 DEBUG [M:0;jenkins-hbase6:36603 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase6:36603 2023-11-19T01:28:53,389 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-19T01:28:53,389 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-19T01:28:53,389 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:53,389 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:53,390 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-19T01:28:53,392 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase6.apache.org,36603,1700357330230 from backup master directory 2023-11-19T01:28:53,392 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-19T01:28:53,396 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase6.apache.org,36603,1700357330230 2023-11-19T01:28:53,396 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:28:53,396 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:28:53,397 WARN [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-19T01:28:53,397 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase6.apache.org,36603,1700357330230 2023-11-19T01:28:53,424 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-11-19T01:28:53,429 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-11-19T01:28:53,508 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/hbase.id with ID: bcd2f70d-bd7e-4f80-847f-ee0a0e0a55fd 2023-11-19T01:28:53,547 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:28:53,562 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:53,562 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:53,605 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:28:53,608 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-11-19T01:28:53,617 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-11-19T01:28:53,622 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-11-19T01:28:53,633 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-11-19T01:28:53,633 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-11-19T01:28:53,634 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.init(RegionGroupingProvider.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createProvider(WALFactory.java:186) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:201) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:28:53,646 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.init(RegionGroupingProvider.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createProvider(WALFactory.java:186) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:201) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:28:53,719 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store 2023-11-19T01:28:53,741 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-11-19T01:28:53,742 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:53,742 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-19T01:28:53,742 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:28:53,742 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:28:53,742 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-19T01:28:53,743 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:28:53,743 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:28:53,743 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:28:53,744 WARN [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/.initializing 2023-11-19T01:28:53,745 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/WALs/jenkins-hbase6.apache.org,36603,1700357330230 2023-11-19T01:28:53,747 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:28:53,754 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase6.apache.org,36603,1700357330230.regiongroup-0 2023-11-19T01:28:53,768 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C36603%2C1700357330230.jenkins-hbase6.apache.org%2C36603%2C1700357330230.regiongroup-0, suffix=, logDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/WALs/jenkins-hbase6.apache.org,36603,1700357330230, archiveDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/oldWALs, maxLogs=10 2023-11-19T01:28:53,799 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/WALs/jenkins-hbase6.apache.org,36603,1700357330230/jenkins-hbase6.apache.org%2C36603%2C1700357330230.jenkins-hbase6.apache.org%2C36603%2C1700357330230.regiongroup-0.1700357333775, exclude list is [], retry=0 2023-11-19T01:28:53,830 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:28:53,837 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:499) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:28:53,908 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/WALs/jenkins-hbase6.apache.org,36603,1700357330230/jenkins-hbase6.apache.org%2C36603%2C1700357330230.jenkins-hbase6.apache.org%2C36603%2C1700357330230.regiongroup-0.1700357333775 2023-11-19T01:28:53,909 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:28:53,909 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:28:53,910 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:53,914 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:28:53,915 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:28:53,983 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:28:54,018 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-11-19T01:28:54,024 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:54,027 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:28:54,028 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:28:54,031 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-11-19T01:28:54,031 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:54,032 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:28:54,032 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:28:54,035 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-11-19T01:28:54,035 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:54,036 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:28:54,036 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:28:54,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-11-19T01:28:54,039 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:54,040 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:28:54,044 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:28:54,045 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:28:54,060 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-11-19T01:28:54,063 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:28:54,067 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:28:54,068 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9608085120, jitterRate=-0.10517734289169312}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-11-19T01:28:54,073 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:28:54,074 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-11-19T01:28:54,108 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba1fb36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:54,162 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(861): No meta location available on zookeeper, skip migrating... 2023-11-19T01:28:54,182 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-11-19T01:28:54,182 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-11-19T01:28:54,185 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-11-19T01:28:54,186 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-11-19T01:28:54,192 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 5 msec 2023-11-19T01:28:54,192 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-11-19T01:28:54,225 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-11-19T01:28:54,240 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-11-19T01:28:54,243 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-11-19T01:28:54,246 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-11-19T01:28:54,247 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-11-19T01:28:54,249 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-11-19T01:28:54,251 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-11-19T01:28:54,258 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-11-19T01:28:54,259 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-11-19T01:28:54,260 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-11-19T01:28:54,262 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-11-19T01:28:54,277 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-11-19T01:28:54,279 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-11-19T01:28:54,284 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-19T01:28:54,284 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-19T01:28:54,285 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:54,285 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:54,285 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(805): Active/primary master=jenkins-hbase6.apache.org,36603,1700357330230, sessionid=0x10123a5ddbf0000, setting cluster-up flag (Was=false) 2023-11-19T01:28:54,294 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-11-19T01:28:54,305 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:54,305 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:54,311 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-11-19T01:28:54,313 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,36603,1700357330230 2023-11-19T01:28:54,317 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:54,317 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:54,322 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-11-19T01:28:54,324 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,36603,1700357330230 2023-11-19T01:28:54,335 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:54,335 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:54,341 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-11-19T01:28:54,342 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,36603,1700357330230 2023-11-19T01:28:54,456 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase6:40967 2023-11-19T01:28:54,458 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(997): ClusterId : bcd2f70d-bd7e-4f80-847f-ee0a0e0a55fd 2023-11-19T01:28:54,460 INFO [RS:0;jenkins-hbase6:40967 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-11-19T01:28:54,462 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-11-19T01:28:54,468 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-11-19T01:28:54,468 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-11-19T01:28:54,473 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-11-19T01:28:54,473 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-11-19T01:28:54,476 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-11-19T01:28:54,477 DEBUG [RS:0;jenkins-hbase6:40967 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bbf81ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:54,478 DEBUG [RS:0;jenkins-hbase6:40967 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a53a0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase6.apache.org/172.31.2.20:0 2023-11-19T01:28:54,483 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-11-19T01:28:54,483 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-11-19T01:28:54,484 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-11-19T01:28:54,484 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-11-19T01:28:54,487 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase6.apache.org,36603,1700357330230 with isa=jenkins-hbase6.apache.org/172.31.2.20:40967, startcode=1700357333274 2023-11-19T01:28:54,497 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-11-19T01:28:54,504 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-11-19T01:28:54,505 DEBUG [RS:0;jenkins-hbase6:40967 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-19T01:28:54,517 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase6.apache.org,36603,1700357330230 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-11-19T01:28:54,523 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:28:54,523 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:28:54,524 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:28:54,524 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:28:54,524 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase6:0, corePoolSize=10, maxPoolSize=10 2023-11-19T01:28:54,524 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:28:54,524 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=2, maxPoolSize=2 2023-11-19T01:28:54,524 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:28:54,531 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1700357364531 2023-11-19T01:28:54,534 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-11-19T01:28:54,544 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-11-19T01:28:54,545 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-11-19T01:28:54,546 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-11-19T01:28:54,552 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:54,552 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-19T01:28:54,558 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-11-19T01:28:54,558 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-11-19T01:28:54,559 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-11-19T01:28:54,559 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-11-19T01:28:54,559 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-11-19T01:28:54,567 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:54,571 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-11-19T01:28:54,579 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-11-19T01:28:54,580 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-11-19T01:28:54,596 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x58e3198a to 127.0.0.1:59682 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:28:54,602 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:45139, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-19T01:28:54,607 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d4f62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:54,607 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-19T01:28:54,609 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-11-19T01:28:54,610 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-11-19T01:28:54,612 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36603 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3212) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:592) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-19T01:28:54,613 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357334612,5,FailOnTimeoutGroup] 2023-11-19T01:28:54,613 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357334613,5,FailOnTimeoutGroup] 2023-11-19T01:28:54,613 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:54,613 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1636): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-11-19T01:28:54,615 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:54,615 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:54,645 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-11-19T01:28:54,646 WARN [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-11-19T01:28:54,747 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase6.apache.org,36603,1700357330230 with isa=jenkins-hbase6.apache.org/172.31.2.20:40967, startcode=1700357333274 2023-11-19T01:28:54,752 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36603 {}] master.ServerManager(396): Registering regionserver=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:28:54,765 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa 2023-11-19T01:28:54,765 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:39587 2023-11-19T01:28:54,765 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=36005 2023-11-19T01:28:54,774 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-11-19T01:28:54,775 DEBUG [RS:0;jenkins-hbase6:40967 {}] zookeeper.ZKUtil(111): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:28:54,775 WARN [RS:0;jenkins-hbase6:40967 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-19T01:28:54,775 INFO [RS:0;jenkins-hbase6:40967 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-11-19T01:28:54,775 INFO [RS:0;jenkins-hbase6:40967 {}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-11-19T01:28:54,775 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:28:54,778 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase6.apache.org,40967,1700357333274] 2023-11-19T01:28:54,798 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-11-19T01:28:54,816 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-11-19T01:28:54,829 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-11-19T01:28:54,834 INFO [RS:0;jenkins-hbase6:40967 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-11-19T01:28:54,834 INFO [RS:0;jenkins-hbase6:40967 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:54,835 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-11-19T01:28:54,844 INFO [RS:0;jenkins-hbase6:40967 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:54,844 DEBUG [RS:0;jenkins-hbase6:40967 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:28:54,844 DEBUG [RS:0;jenkins-hbase6:40967 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:28:54,844 DEBUG [RS:0;jenkins-hbase6:40967 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:28:54,845 DEBUG [RS:0;jenkins-hbase6:40967 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:28:54,845 DEBUG [RS:0;jenkins-hbase6:40967 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:28:54,845 DEBUG [RS:0;jenkins-hbase6:40967 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase6:0, corePoolSize=2, maxPoolSize=2 2023-11-19T01:28:54,845 DEBUG [RS:0;jenkins-hbase6:40967 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:28:54,845 DEBUG [RS:0;jenkins-hbase6:40967 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:28:54,845 DEBUG [RS:0;jenkins-hbase6:40967 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:28:54,845 DEBUG [RS:0;jenkins-hbase6:40967 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:28:54,846 DEBUG [RS:0;jenkins-hbase6:40967 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:28:54,846 DEBUG [RS:0;jenkins-hbase6:40967 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0, corePoolSize=3, maxPoolSize=3 2023-11-19T01:28:54,846 DEBUG [RS:0;jenkins-hbase6:40967 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0, corePoolSize=3, maxPoolSize=3 2023-11-19T01:28:54,847 INFO [RS:0;jenkins-hbase6:40967 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:54,847 INFO [RS:0;jenkins-hbase6:40967 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:54,848 INFO [RS:0;jenkins-hbase6:40967 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:54,848 INFO [RS:0;jenkins-hbase6:40967 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:54,848 INFO [RS:0;jenkins-hbase6:40967 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,40967,1700357333274-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-19T01:28:54,863 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-11-19T01:28:54,866 INFO [RS:0;jenkins-hbase6:40967 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,40967,1700357333274-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:54,881 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.Replication(204): jenkins-hbase6.apache.org,40967,1700357333274 started 2023-11-19T01:28:54,881 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase6.apache.org,40967,1700357333274, RpcServer on jenkins-hbase6.apache.org/172.31.2.20:40967, sessionid=0x10123a5ddbf0001 2023-11-19T01:28:54,882 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-11-19T01:28:54,882 DEBUG [RS:0;jenkins-hbase6:40967 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:28:54,882 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,40967,1700357333274' 2023-11-19T01:28:54,882 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-11-19T01:28:54,883 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-11-19T01:28:54,883 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-11-19T01:28:54,883 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-11-19T01:28:54,884 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,40967,1700357333274' 2023-11-19T01:28:54,884 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:28:54,884 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:28:54,885 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-11-19T01:28:54,885 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-11-19T01:28:54,885 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-11-19T01:28:54,885 DEBUG [RS:0;jenkins-hbase6:40967 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:28:54,885 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,40967,1700357333274' 2023-11-19T01:28:54,885 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-11-19T01:28:54,885 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-11-19T01:28:54,886 DEBUG [RS:0;jenkins-hbase6:40967 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-11-19T01:28:54,886 INFO [RS:0;jenkins-hbase6:40967 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-11-19T01:28:54,886 INFO [RS:0;jenkins-hbase6:40967 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-11-19T01:28:54,996 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-11-19T01:28:54,996 INFO [RS:0;jenkins-hbase6:40967 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:28:54,996 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa 2023-11-19T01:28:54,997 INFO [RS:0;jenkins-hbase6:40967 {}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase6.apache.org,40967,1700357333274.regiongroup-0 2023-11-19T01:28:54,999 INFO [RS:0;jenkins-hbase6:40967 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0, suffix=, logDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274, archiveDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs, maxLogs=32 2023-11-19T01:28:55,016 DEBUG [RS:0;jenkins-hbase6:40967 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357335002, exclude list is [], retry=0 2023-11-19T01:28:55,021 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:28:55,025 INFO [RS:0;jenkins-hbase6:40967 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357335002 2023-11-19T01:28:55,026 DEBUG [RS:0;jenkins-hbase6:40967 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:28:55,410 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:55,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-19T01:28:55,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-19T01:28:55,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:55,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:28:55,417 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-19T01:28:55,419 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-19T01:28:55,419 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:55,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:28:55,420 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-19T01:28:55,422 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-19T01:28:55,422 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:55,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:28:55,424 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740 2023-11-19T01:28:55,425 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740 2023-11-19T01:28:55,427 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-19T01:28:55,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-19T01:28:55,432 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:28:55,433 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11070189440, jitterRate=0.030991733074188232}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-19T01:28:55,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-19T01:28:55,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-19T01:28:55,435 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-19T01:28:55,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-19T01:28:55,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-19T01:28:55,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-19T01:28:55,437 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-19T01:28:55,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-19T01:28:55,441 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-11-19T01:28:55,441 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-11-19T01:28:55,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-11-19T01:28:55,462 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-11-19T01:28:55,467 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-11-19T01:28:55,619 DEBUG [jenkins-hbase6:36603 {}] assignment.AssignmentManager(2319): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-11-19T01:28:55,628 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:28:55,633 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase6.apache.org,40967,1700357333274, state=OPENING 2023-11-19T01:28:55,641 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-11-19T01:28:55,643 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:55,643 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:55,644 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:28:55,644 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:28:55,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:28:55,985 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:28:55,987 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-11-19T01:28:55,991 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37748, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-11-19T01:28:56,003 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-11-19T01:28:56,003 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-11-19T01:28:56,003 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-11-19T01:28:56,003 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:28:56,004 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-11-19T01:28:56,007 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta, suffix=.meta, logDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274, archiveDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs, maxLogs=32 2023-11-19T01:28:56,027 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357336010.meta, exclude list is [], retry=0 2023-11-19T01:28:56,032 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:28:56,036 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357336010.meta 2023-11-19T01:28:56,036 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:28:56,036 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:28:56,043 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:28:56,044 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-11-19T01:28:56,062 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-11-19T01:28:56,062 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-11-19T01:28:56,067 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-11-19T01:28:56,067 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:56,068 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-11-19T01:28:56,068 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-11-19T01:28:56,071 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-19T01:28:56,072 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-19T01:28:56,072 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:56,073 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:28:56,073 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-19T01:28:56,074 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-19T01:28:56,074 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:56,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:28:56,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-19T01:28:56,077 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-19T01:28:56,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:56,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:28:56,079 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740 2023-11-19T01:28:56,082 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740 2023-11-19T01:28:56,085 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-19T01:28:56,088 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-19T01:28:56,089 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9825059360, jitterRate=-0.08497004210948944}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-19T01:28:56,091 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-19T01:28:56,103 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1700357335976 2023-11-19T01:28:56,123 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-11-19T01:28:56,124 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-11-19T01:28:56,125 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:28:56,127 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase6.apache.org,40967,1700357333274, state=OPEN 2023-11-19T01:28:56,131 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-19T01:28:56,131 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-19T01:28:56,131 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:28:56,131 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:28:56,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-11-19T01:28:56,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase6.apache.org,40967,1700357333274 in 484 msec 2023-11-19T01:28:56,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-11-19T01:28:56,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 687 msec 2023-11-19T01:28:56,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.7450 sec 2023-11-19T01:28:56,151 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1064): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1700357336150, completionTime=-1 2023-11-19T01:28:56,151 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ServerManager(816): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-11-19T01:28:56,151 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1645): Joining cluster... 2023-11-19T01:28:56,228 DEBUG [hconnection-0x5b0d6c55-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:56,232 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37754, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:56,249 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1657): Number of RegionServers=1 2023-11-19T01:28:56,250 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1700357396250 2023-11-19T01:28:56,250 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1700357456250 2023-11-19T01:28:56,250 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1664): Joined the cluster in 98 msec 2023-11-19T01:28:56,273 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,36603,1700357330230-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:56,274 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,36603,1700357330230-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:56,274 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,36603,1700357330230-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:56,276 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase6:36603, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:56,276 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-11-19T01:28:56,282 DEBUG [master/jenkins-hbase6:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-11-19T01:28:56,297 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-11-19T01:28:56,298 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(2369): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-19T01:28:56,307 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-11-19T01:28:56,311 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:28:56,313 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:56,315 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:28:56,734 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 36f383735ae1fecc54a98975fe8d38af, NAME => 'hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa 2023-11-19T01:28:57,146 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:57,146 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 36f383735ae1fecc54a98975fe8d38af, disabling compactions & flushes 2023-11-19T01:28:57,146 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. 2023-11-19T01:28:57,146 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. 2023-11-19T01:28:57,146 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. after waiting 0 ms 2023-11-19T01:28:57,146 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. 2023-11-19T01:28:57,146 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. 2023-11-19T01:28:57,146 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 36f383735ae1fecc54a98975fe8d38af: 2023-11-19T01:28:57,148 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:28:57,164 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1700357337150"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357337150"}]},"ts":"1700357337150"} 2023-11-19T01:28:57,196 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:28:57,197 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:28:57,203 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357337198"}]},"ts":"1700357337198"} 2023-11-19T01:28:57,206 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-11-19T01:28:57,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=36f383735ae1fecc54a98975fe8d38af, ASSIGN}] 2023-11-19T01:28:57,215 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=36f383735ae1fecc54a98975fe8d38af, ASSIGN 2023-11-19T01:28:57,217 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=36f383735ae1fecc54a98975fe8d38af, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,40967,1700357333274; forceNewPlan=false, retain=false 2023-11-19T01:28:57,367 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=36f383735ae1fecc54a98975fe8d38af, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:28:57,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 36f383735ae1fecc54a98975fe8d38af, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:28:57,525 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:28:57,531 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. 2023-11-19T01:28:57,531 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:28:57,532 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase6.apache.org,40967,1700357333274.regiongroup-1 2023-11-19T01:28:57,534 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1, suffix=, logDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274, archiveDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs, maxLogs=32 2023-11-19T01:28:57,547 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357337536, exclude list is [], retry=0 2023-11-19T01:28:57,551 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:28:57,554 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357337536 2023-11-19T01:28:57,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:28:57,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 36f383735ae1fecc54a98975fe8d38af, NAME => 'hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:28:57,556 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:28:57,556 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 36f383735ae1fecc54a98975fe8d38af 2023-11-19T01:28:57,556 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:57,556 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 36f383735ae1fecc54a98975fe8d38af 2023-11-19T01:28:57,556 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 36f383735ae1fecc54a98975fe8d38af 2023-11-19T01:28:57,558 INFO [StoreOpener-36f383735ae1fecc54a98975fe8d38af-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 36f383735ae1fecc54a98975fe8d38af 2023-11-19T01:28:57,560 INFO [StoreOpener-36f383735ae1fecc54a98975fe8d38af-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36f383735ae1fecc54a98975fe8d38af columnFamilyName info 2023-11-19T01:28:57,560 DEBUG [StoreOpener-36f383735ae1fecc54a98975fe8d38af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:57,561 INFO [StoreOpener-36f383735ae1fecc54a98975fe8d38af-1 {}] regionserver.HStore(324): Store=36f383735ae1fecc54a98975fe8d38af/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:28:57,562 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/namespace/36f383735ae1fecc54a98975fe8d38af 2023-11-19T01:28:57,563 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/namespace/36f383735ae1fecc54a98975fe8d38af 2023-11-19T01:28:57,566 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 36f383735ae1fecc54a98975fe8d38af 2023-11-19T01:28:57,570 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/namespace/36f383735ae1fecc54a98975fe8d38af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:28:57,571 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 36f383735ae1fecc54a98975fe8d38af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9876394560, jitterRate=-0.08018907904624939}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:28:57,573 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 36f383735ae1fecc54a98975fe8d38af: 2023-11-19T01:28:57,574 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af., pid=6, masterSystemTime=1700357337525 2023-11-19T01:28:57,578 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. 2023-11-19T01:28:57,578 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. 2023-11-19T01:28:57,579 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=36f383735ae1fecc54a98975fe8d38af, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:28:57,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-11-19T01:28:57,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 36f383735ae1fecc54a98975fe8d38af, server=jenkins-hbase6.apache.org,40967,1700357333274 in 214 msec 2023-11-19T01:28:57,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-11-19T01:28:57,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=36f383735ae1fecc54a98975fe8d38af, ASSIGN in 376 msec 2023-11-19T01:28:57,592 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:28:57,593 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357337592"}]},"ts":"1700357337592"} 2023-11-19T01:28:57,595 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-11-19T01:28:57,599 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:28:57,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.3000 sec 2023-11-19T01:28:57,611 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-11-19T01:28:57,613 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-11-19T01:28:57,613 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:57,613 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:28:57,661 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-11-19T01:28:57,687 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:28:57,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 40 msec 2023-11-19T01:28:57,705 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-11-19T01:28:57,716 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:28:57,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 14 msec 2023-11-19T01:28:57,732 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-11-19T01:28:57,735 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-11-19T01:28:57,735 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1194): Master has completed initialization 4.338sec 2023-11-19T01:28:57,737 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-11-19T01:28:57,738 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-11-19T01:28:57,740 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-11-19T01:28:57,740 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-11-19T01:28:57,741 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-11-19T01:28:57,742 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,36603,1700357330230-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-19T01:28:57,742 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,36603,1700357330230-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-11-19T01:28:57,755 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1297): Balancer post startup initialization complete, took 0 seconds 2023-11-19T01:28:57,757 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-11-19T01:28:57,801 DEBUG [Listener at localhost/35467 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x68b91fb6 to 127.0.0.1:59682 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:28:57,805 DEBUG [Listener at localhost/35467 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553933e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:57,810 DEBUG [Listener at localhost/35467 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-11-19T01:28:57,810 DEBUG [Listener at localhost/35467 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-11-19T01:28:57,825 DEBUG [hconnection-0x625798ee-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:57,836 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37770, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:57,847 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase6.apache.org,36603,1700357330230 2023-11-19T01:28:57,847 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-11-19T01:28:57,847 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/test.cache.data in system properties and HBase conf 2023-11-19T01:28:57,847 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop.tmp.dir in system properties and HBase conf 2023-11-19T01:28:57,847 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop.log.dir in system properties and HBase conf 2023-11-19T01:28:57,847 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-19T01:28:57,847 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-19T01:28:57,847 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-19T01:28:57,848 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:28:57,848 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:28:57,848 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-19T01:28:57,848 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:28:57,848 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-19T01:28:57,848 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-19T01:28:57,848 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:28:57,848 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:28:57,848 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-19T01:28:57,848 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/nfs.dump.dir in system properties and HBase conf 2023-11-19T01:28:57,848 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/java.io.tmpdir in system properties and HBase conf 2023-11-19T01:28:57,848 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:28:57,848 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-19T01:28:57,849 INFO [Listener at localhost/35467 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-11-19T01:28:58,951 WARN [Thread-251 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:28:59,313 INFO [Thread-251 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:28:59,319 INFO [Thread-251 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_44965_jobhistory____.gp2qjl/webapp 2023-11-19T01:28:59,478 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-11-19T01:28:59,478 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-11-19T01:28:59,478 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:28:59,481 INFO [Thread-251 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:28:59,573 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:28:59,701 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:28:59,987 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-11-19T01:29:00,020 INFO [Thread-251 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:44965 2023-11-19T01:29:00,449 WARN [Listener at jenkins-hbase6.apache.org/34011 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:29:00,528 WARN [Listener at jenkins-hbase6.apache.org/34011 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:29:00,529 INFO [Listener at jenkins-hbase6.apache.org/34011 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:29:00,535 INFO [Listener at jenkins-hbase6.apache.org/34011 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_33581_cluster____.5hau94/webapp 2023-11-19T01:29:00,604 INFO [Listener at jenkins-hbase6.apache.org/34011 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:29:00,605 INFO [Listener at jenkins-hbase6.apache.org/34011 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-11-19T01:29:00,605 INFO [Listener at jenkins-hbase6.apache.org/34011 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:29:00,605 INFO [Listener at jenkins-hbase6.apache.org/34011 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:29:00,615 INFO [Listener at jenkins-hbase6.apache.org/34011 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:29:00,807 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-11-19T01:29:00,832 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-11-19T01:29:00,855 INFO [Listener at jenkins-hbase6.apache.org/34011 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:29:00,964 INFO [Listener at jenkins-hbase6.apache.org/34011 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:29:00,973 INFO [Listener at jenkins-hbase6.apache.org/34011 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:33581 2023-11-19T01:29:01,318 WARN [Listener at jenkins-hbase6.apache.org/39627 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:29:01,342 WARN [Listener at jenkins-hbase6.apache.org/39627 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:29:01,342 INFO [Listener at jenkins-hbase6.apache.org/39627 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:29:01,348 INFO [Listener at jenkins-hbase6.apache.org/39627 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_44751_node____fc1ibz/webapp 2023-11-19T01:29:01,402 INFO [Listener at jenkins-hbase6.apache.org/39627 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-19T01:29:01,402 INFO [Listener at jenkins-hbase6.apache.org/39627 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:29:01,403 INFO [Listener at jenkins-hbase6.apache.org/39627 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:29:01,403 INFO [Listener at jenkins-hbase6.apache.org/39627 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:29:01,412 INFO [Listener at jenkins-hbase6.apache.org/39627 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:29:01,436 INFO [Listener at jenkins-hbase6.apache.org/39627 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:29:01,503 INFO [Listener at jenkins-hbase6.apache.org/39627 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:29:01,513 INFO [Listener at jenkins-hbase6.apache.org/39627 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:44751 2023-11-19T01:29:01,607 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:29:01,700 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-11-19T01:29:01,700 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-11-19T01:29:01,703 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-11-19T01:29:01,703 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-11-19T01:29:01,708 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:01,708 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-11-19T01:29:01,708 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-19T01:29:01,708 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-11-19T01:29:01,708 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-11-19T01:29:01,708 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-11-19T01:29:01,760 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:29:01,769 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:29:01,770 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:29:01,779 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_45323_node____f9zm6n/webapp 2023-11-19T01:29:01,840 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-19T01:29:01,841 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:29:01,841 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:29:01,841 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:29:01,850 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:29:01,870 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:29:01,936 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:29:01,944 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:45323 2023-11-19T01:29:01,974 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-11-19T01:29:01,974 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:39587/backupUT 2023-11-19T01:29:01,986 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:01,990 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:47230, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:01,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns1'} 2023-11-19T01:29:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-11-19T01:29:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-19T01:29:02,028 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:29:02,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 33 msec 2023-11-19T01:29:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-19T01:29:02,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns2'} 2023-11-19T01:29:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-11-19T01:29:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-19T01:29:02,139 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:29:02,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 18 msec 2023-11-19T01:29:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-19T01:29:02,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns3'} 2023-11-19T01:29:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-11-19T01:29:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-19T01:29:02,249 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:29:02,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 20 msec 2023-11-19T01:29:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-19T01:29:02,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns4'} 2023-11-19T01:29:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-11-19T01:29:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-19T01:29:02,360 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:29:02,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 17 msec 2023-11-19T01:29:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-19T01:29:02,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'test-1700357341974', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:02,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1700357341974 2023-11-19T01:29:02,476 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1700357341974 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:29:02,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "test-1700357341974" procId is: 13 2023-11-19T01:29:02,477 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:02,478 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1700357341974 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:29:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:29:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:29:02,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:29:02,919 INFO [RegionOpenAndInit-test-1700357341974-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3c282db26b60e0d893a6e12adb946977, NAME => 'test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1700357341974', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa 2023-11-19T01:29:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:29:03,339 DEBUG [RegionOpenAndInit-test-1700357341974-pool-0 {}] regionserver.HRegion(888): Instantiated test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:03,339 DEBUG [RegionOpenAndInit-test-1700357341974-pool-0 {}] regionserver.HRegion(1668): Closing 3c282db26b60e0d893a6e12adb946977, disabling compactions & flushes 2023-11-19T01:29:03,340 INFO [RegionOpenAndInit-test-1700357341974-pool-0 {}] regionserver.HRegion(1690): Closing region test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:03,340 DEBUG [RegionOpenAndInit-test-1700357341974-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:03,340 DEBUG [RegionOpenAndInit-test-1700357341974-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. after waiting 0 ms 2023-11-19T01:29:03,340 DEBUG [RegionOpenAndInit-test-1700357341974-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:03,340 INFO [RegionOpenAndInit-test-1700357341974-pool-0 {}] regionserver.HRegion(1909): Closed test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:03,340 DEBUG [RegionOpenAndInit-test-1700357341974-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3c282db26b60e0d893a6e12adb946977: 2023-11-19T01:29:03,342 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1700357341974 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:29:03,342 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1700357343342"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357343342"}]},"ts":"1700357343342"} 2023-11-19T01:29:03,345 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:29:03,346 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1700357341974 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:29:03,346 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357341974","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357343346"}]},"ts":"1700357343346"} 2023-11-19T01:29:03,349 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357341974, state=ENABLING in hbase:meta 2023-11-19T01:29:03,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1700357341974, region=3c282db26b60e0d893a6e12adb946977, ASSIGN}] 2023-11-19T01:29:03,359 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1700357341974, region=3c282db26b60e0d893a6e12adb946977, ASSIGN 2023-11-19T01:29:03,360 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1700357341974, region=3c282db26b60e0d893a6e12adb946977, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,40967,1700357333274; forceNewPlan=false, retain=false 2023-11-19T01:29:03,512 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=3c282db26b60e0d893a6e12adb946977, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:03,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 3c282db26b60e0d893a6e12adb946977, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:29:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:29:03,668 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:03,675 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:03,675 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 3c282db26b60e0d893a6e12adb946977, NAME => 'test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:29:03,676 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:29:03,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1700357341974 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:03,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:03,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:03,676 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:03,679 INFO [StoreOpener-3c282db26b60e0d893a6e12adb946977-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:03,682 INFO [StoreOpener-3c282db26b60e0d893a6e12adb946977-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c282db26b60e0d893a6e12adb946977 columnFamilyName f 2023-11-19T01:29:03,682 DEBUG [StoreOpener-3c282db26b60e0d893a6e12adb946977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:03,683 INFO [StoreOpener-3c282db26b60e0d893a6e12adb946977-1 {}] regionserver.HStore(324): Store=3c282db26b60e0d893a6e12adb946977/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:03,684 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:03,684 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:03,688 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:03,694 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:29:03,695 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 3c282db26b60e0d893a6e12adb946977; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12017231840, jitterRate=0.11919192969799042}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:29:03,696 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 3c282db26b60e0d893a6e12adb946977: 2023-11-19T01:29:03,697 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977., pid=15, masterSystemTime=1700357343668 2023-11-19T01:29:03,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:03,700 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:03,701 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=3c282db26b60e0d893a6e12adb946977, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:03,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-11-19T01:29:03,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 3c282db26b60e0d893a6e12adb946977, server=jenkins-hbase6.apache.org,40967,1700357333274 in 188 msec 2023-11-19T01:29:03,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-11-19T01:29:03,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1700357341974, region=3c282db26b60e0d893a6e12adb946977, ASSIGN in 349 msec 2023-11-19T01:29:03,710 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1700357341974 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:29:03,711 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357341974","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357343710"}]},"ts":"1700357343710"} 2023-11-19T01:29:03,712 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357341974, state=ENABLED in hbase:meta 2023-11-19T01:29:03,715 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1700357341974 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:29:03,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1700357341974 in 1.2450 sec 2023-11-19T01:29:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:29:04,597 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: default:test-1700357341974, procId: 13 completed 2023-11-19T01:29:04,599 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x344d9ed9 to 127.0.0.1:59682 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:29:04,604 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b376e40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:04,607 DEBUG [hconnection-0x51cd22aa-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:04,609 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:42150, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:04,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40967 {}] regionserver.HRegion(8210): writing data to region test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. with WAL disabled. Data may be lost in the event of a crash. 2023-11-19T01:29:04,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns2:test-17003573419741', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-17003573419741 2023-11-19T01:29:04,710 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-17003573419741 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:29:04,710 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:04,710 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns2" qualifier: "test-17003573419741" procId is: 16 2023-11-19T01:29:04,711 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-17003573419741 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:29:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:29:04,725 INFO [RegionOpenAndInit-ns2:test-17003573419741-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 4626e266aa32d68d09dea77ed5ea495c, NAME => 'ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17003573419741', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa 2023-11-19T01:29:04,739 DEBUG [RegionOpenAndInit-ns2:test-17003573419741-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:04,739 DEBUG [RegionOpenAndInit-ns2:test-17003573419741-pool-0 {}] regionserver.HRegion(1668): Closing 4626e266aa32d68d09dea77ed5ea495c, disabling compactions & flushes 2023-11-19T01:29:04,739 INFO [RegionOpenAndInit-ns2:test-17003573419741-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:29:04,739 DEBUG [RegionOpenAndInit-ns2:test-17003573419741-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:29:04,739 DEBUG [RegionOpenAndInit-ns2:test-17003573419741-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. after waiting 0 ms 2023-11-19T01:29:04,739 DEBUG [RegionOpenAndInit-ns2:test-17003573419741-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:29:04,740 INFO [RegionOpenAndInit-ns2:test-17003573419741-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:29:04,740 DEBUG [RegionOpenAndInit-ns2:test-17003573419741-pool-0 {}] regionserver.HRegion(1622): Region close journal for 4626e266aa32d68d09dea77ed5ea495c: 2023-11-19T01:29:04,741 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-17003573419741 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:29:04,741 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357344741"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357344741"}]},"ts":"1700357344741"} 2023-11-19T01:29:04,743 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:29:04,745 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-17003573419741 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:29:04,745 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003573419741","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357344745"}]},"ts":"1700357344745"} 2023-11-19T01:29:04,746 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003573419741, state=ENABLING in hbase:meta 2023-11-19T01:29:04,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-17003573419741, region=4626e266aa32d68d09dea77ed5ea495c, ASSIGN}] 2023-11-19T01:29:04,753 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-17003573419741, region=4626e266aa32d68d09dea77ed5ea495c, ASSIGN 2023-11-19T01:29:04,754 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-17003573419741, region=4626e266aa32d68d09dea77ed5ea495c, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,40967,1700357333274; forceNewPlan=false, retain=false 2023-11-19T01:29:04,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:29:04,905 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=4626e266aa32d68d09dea77ed5ea495c, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:04,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 4626e266aa32d68d09dea77ed5ea495c, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:29:05,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:29:05,061 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:05,066 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:29:05,066 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 4626e266aa32d68d09dea77ed5ea495c, NAME => 'ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:29:05,066 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:29:05,066 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003573419741 4626e266aa32d68d09dea77ed5ea495c 2023-11-19T01:29:05,067 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:05,067 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 4626e266aa32d68d09dea77ed5ea495c 2023-11-19T01:29:05,067 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 4626e266aa32d68d09dea77ed5ea495c 2023-11-19T01:29:05,069 INFO [StoreOpener-4626e266aa32d68d09dea77ed5ea495c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 4626e266aa32d68d09dea77ed5ea495c 2023-11-19T01:29:05,071 INFO [StoreOpener-4626e266aa32d68d09dea77ed5ea495c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4626e266aa32d68d09dea77ed5ea495c columnFamilyName f 2023-11-19T01:29:05,071 DEBUG [StoreOpener-4626e266aa32d68d09dea77ed5ea495c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:05,072 INFO [StoreOpener-4626e266aa32d68d09dea77ed5ea495c-1 {}] regionserver.HStore(324): Store=4626e266aa32d68d09dea77ed5ea495c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:05,073 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c 2023-11-19T01:29:05,073 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c 2023-11-19T01:29:05,077 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 4626e266aa32d68d09dea77ed5ea495c 2023-11-19T01:29:05,084 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:29:05,090 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 4626e266aa32d68d09dea77ed5ea495c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11626692000, jitterRate=0.08282007277011871}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:29:05,091 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 4626e266aa32d68d09dea77ed5ea495c: 2023-11-19T01:29:05,093 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c., pid=18, masterSystemTime=1700357345061 2023-11-19T01:29:05,095 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:29:05,095 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:29:05,096 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=4626e266aa32d68d09dea77ed5ea495c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:05,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-11-19T01:29:05,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 4626e266aa32d68d09dea77ed5ea495c, server=jenkins-hbase6.apache.org,40967,1700357333274 in 191 msec 2023-11-19T01:29:05,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-11-19T01:29:05,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-17003573419741, region=4626e266aa32d68d09dea77ed5ea495c, ASSIGN in 351 msec 2023-11-19T01:29:05,106 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-17003573419741 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:29:05,106 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003573419741","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357345106"}]},"ts":"1700357345106"} 2023-11-19T01:29:05,108 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003573419741, state=ENABLED in hbase:meta 2023-11-19T01:29:05,112 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-17003573419741 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:29:05,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-17003573419741 in 406 msec 2023-11-19T01:29:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:29:05,317 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns2:test-17003573419741, procId: 16 completed 2023-11-19T01:29:05,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40967 {}] regionserver.HRegion(8210): writing data to region ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. with WAL disabled. Data may be lost in the event of a crash. 2023-11-19T01:29:05,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns3:test-17003573419742', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-17003573419742 2023-11-19T01:29:05,414 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-17003573419742 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:29:05,414 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:05,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns3" qualifier: "test-17003573419742" procId is: 19 2023-11-19T01:29:05,415 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-17003573419742 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:29:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:29:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:29:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:29:05,831 INFO [RegionOpenAndInit-ns3:test-17003573419742-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 587767aa1f479d2c3ca4e5abbe608c38, NAME => 'ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17003573419742', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa 2023-11-19T01:29:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:29:06,245 DEBUG [RegionOpenAndInit-ns3:test-17003573419742-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:06,245 DEBUG [RegionOpenAndInit-ns3:test-17003573419742-pool-0 {}] regionserver.HRegion(1668): Closing 587767aa1f479d2c3ca4e5abbe608c38, disabling compactions & flushes 2023-11-19T01:29:06,245 INFO [RegionOpenAndInit-ns3:test-17003573419742-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. 2023-11-19T01:29:06,245 DEBUG [RegionOpenAndInit-ns3:test-17003573419742-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. 2023-11-19T01:29:06,245 DEBUG [RegionOpenAndInit-ns3:test-17003573419742-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. after waiting 0 ms 2023-11-19T01:29:06,245 DEBUG [RegionOpenAndInit-ns3:test-17003573419742-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. 2023-11-19T01:29:06,245 INFO [RegionOpenAndInit-ns3:test-17003573419742-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. 2023-11-19T01:29:06,246 DEBUG [RegionOpenAndInit-ns3:test-17003573419742-pool-0 {}] regionserver.HRegion(1622): Region close journal for 587767aa1f479d2c3ca4e5abbe608c38: 2023-11-19T01:29:06,247 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-17003573419742 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:29:06,248 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357346247"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357346247"}]},"ts":"1700357346247"} 2023-11-19T01:29:06,250 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:29:06,251 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-17003573419742 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:29:06,251 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003573419742","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357346251"}]},"ts":"1700357346251"} 2023-11-19T01:29:06,253 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003573419742, state=ENABLING in hbase:meta 2023-11-19T01:29:06,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-17003573419742, region=587767aa1f479d2c3ca4e5abbe608c38, ASSIGN}] 2023-11-19T01:29:06,263 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-17003573419742, region=587767aa1f479d2c3ca4e5abbe608c38, ASSIGN 2023-11-19T01:29:06,264 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-17003573419742, region=587767aa1f479d2c3ca4e5abbe608c38, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,40967,1700357333274; forceNewPlan=false, retain=false 2023-11-19T01:29:06,416 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=587767aa1f479d2c3ca4e5abbe608c38, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:06,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 587767aa1f479d2c3ca4e5abbe608c38, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:29:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:29:06,571 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:06,576 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. 2023-11-19T01:29:06,576 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 587767aa1f479d2c3ca4e5abbe608c38, NAME => 'ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:29:06,576 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:29:06,576 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003573419742 587767aa1f479d2c3ca4e5abbe608c38 2023-11-19T01:29:06,576 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:06,576 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 587767aa1f479d2c3ca4e5abbe608c38 2023-11-19T01:29:06,577 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 587767aa1f479d2c3ca4e5abbe608c38 2023-11-19T01:29:06,578 INFO [StoreOpener-587767aa1f479d2c3ca4e5abbe608c38-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 587767aa1f479d2c3ca4e5abbe608c38 2023-11-19T01:29:06,580 INFO [StoreOpener-587767aa1f479d2c3ca4e5abbe608c38-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 587767aa1f479d2c3ca4e5abbe608c38 columnFamilyName f 2023-11-19T01:29:06,580 DEBUG [StoreOpener-587767aa1f479d2c3ca4e5abbe608c38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:06,581 INFO [StoreOpener-587767aa1f479d2c3ca4e5abbe608c38-1 {}] regionserver.HStore(324): Store=587767aa1f479d2c3ca4e5abbe608c38/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:06,582 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns3/test-17003573419742/587767aa1f479d2c3ca4e5abbe608c38 2023-11-19T01:29:06,582 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns3/test-17003573419742/587767aa1f479d2c3ca4e5abbe608c38 2023-11-19T01:29:06,585 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 587767aa1f479d2c3ca4e5abbe608c38 2023-11-19T01:29:06,589 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns3/test-17003573419742/587767aa1f479d2c3ca4e5abbe608c38/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:29:06,590 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 587767aa1f479d2c3ca4e5abbe608c38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11058090240, jitterRate=0.029864907264709473}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:29:06,591 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 587767aa1f479d2c3ca4e5abbe608c38: 2023-11-19T01:29:06,592 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38., pid=21, masterSystemTime=1700357346571 2023-11-19T01:29:06,595 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. 2023-11-19T01:29:06,595 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. 2023-11-19T01:29:06,596 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=587767aa1f479d2c3ca4e5abbe608c38, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:06,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-11-19T01:29:06,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 587767aa1f479d2c3ca4e5abbe608c38, server=jenkins-hbase6.apache.org,40967,1700357333274 in 181 msec 2023-11-19T01:29:06,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-11-19T01:29:06,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-17003573419742, region=587767aa1f479d2c3ca4e5abbe608c38, ASSIGN in 340 msec 2023-11-19T01:29:06,605 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-17003573419742 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:29:06,605 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003573419742","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357346605"}]},"ts":"1700357346605"} 2023-11-19T01:29:06,608 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003573419742, state=ENABLED in hbase:meta 2023-11-19T01:29:06,611 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-17003573419742 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:29:06,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-17003573419742 in 1.2010 sec 2023-11-19T01:29:06,835 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17003573419742' 2023-11-19T01:29:06,836 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1700357341974' 2023-11-19T01:29:06,836 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17003573419741' 2023-11-19T01:29:07,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:29:07,524 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns3:test-17003573419742, procId: 19 completed 2023-11-19T01:29:07,524 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-17003573419742 get assigned. Timeout = 60000ms 2023-11-19T01:29:07,526 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:29:07,536 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-17003573419742 assigned to meta. Checking AM states. 2023-11-19T01:29:07,537 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:29:07,537 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-17003573419742 assigned. 2023-11-19T01:29:07,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns4:test-17003573419743', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-17003573419743 2023-11-19T01:29:07,542 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-17003573419743 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:29:07,542 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:07,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns4" qualifier: "test-17003573419743" procId is: 22 2023-11-19T01:29:07,543 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-17003573419743 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:29:07,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:29:07,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:29:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:29:07,957 INFO [RegionOpenAndInit-ns4:test-17003573419743-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 4b19ffa2d6c4736cdf4abc8f26e2bae3, NAME => 'ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17003573419743', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa 2023-11-19T01:29:07,969 DEBUG [RegionOpenAndInit-ns4:test-17003573419743-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:07,969 DEBUG [RegionOpenAndInit-ns4:test-17003573419743-pool-0 {}] regionserver.HRegion(1668): Closing 4b19ffa2d6c4736cdf4abc8f26e2bae3, disabling compactions & flushes 2023-11-19T01:29:07,969 INFO [RegionOpenAndInit-ns4:test-17003573419743-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. 2023-11-19T01:29:07,969 DEBUG [RegionOpenAndInit-ns4:test-17003573419743-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. 2023-11-19T01:29:07,969 DEBUG [RegionOpenAndInit-ns4:test-17003573419743-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. after waiting 0 ms 2023-11-19T01:29:07,969 DEBUG [RegionOpenAndInit-ns4:test-17003573419743-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. 2023-11-19T01:29:07,969 INFO [RegionOpenAndInit-ns4:test-17003573419743-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. 2023-11-19T01:29:07,969 DEBUG [RegionOpenAndInit-ns4:test-17003573419743-pool-0 {}] regionserver.HRegion(1622): Region close journal for 4b19ffa2d6c4736cdf4abc8f26e2bae3: 2023-11-19T01:29:07,971 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-17003573419743 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:29:07,971 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357347971"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357347971"}]},"ts":"1700357347971"} 2023-11-19T01:29:07,973 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:29:07,975 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-17003573419743 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:29:07,975 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-17003573419743","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357347975"}]},"ts":"1700357347975"} 2023-11-19T01:29:07,977 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-17003573419743, state=ENABLING in hbase:meta 2023-11-19T01:29:07,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-17003573419743, region=4b19ffa2d6c4736cdf4abc8f26e2bae3, ASSIGN}] 2023-11-19T01:29:07,984 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-17003573419743, region=4b19ffa2d6c4736cdf4abc8f26e2bae3, ASSIGN 2023-11-19T01:29:07,985 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-17003573419743, region=4b19ffa2d6c4736cdf4abc8f26e2bae3, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,40967,1700357333274; forceNewPlan=false, retain=false 2023-11-19T01:29:08,136 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=4b19ffa2d6c4736cdf4abc8f26e2bae3, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:08,139 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 4b19ffa2d6c4736cdf4abc8f26e2bae3, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:29:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:29:08,291 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:08,296 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. 2023-11-19T01:29:08,296 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 4b19ffa2d6c4736cdf4abc8f26e2bae3, NAME => 'ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:29:08,297 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:29:08,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003573419743 4b19ffa2d6c4736cdf4abc8f26e2bae3 2023-11-19T01:29:08,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:08,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 4b19ffa2d6c4736cdf4abc8f26e2bae3 2023-11-19T01:29:08,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 4b19ffa2d6c4736cdf4abc8f26e2bae3 2023-11-19T01:29:08,299 INFO [StoreOpener-4b19ffa2d6c4736cdf4abc8f26e2bae3-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 4b19ffa2d6c4736cdf4abc8f26e2bae3 2023-11-19T01:29:08,301 INFO [StoreOpener-4b19ffa2d6c4736cdf4abc8f26e2bae3-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b19ffa2d6c4736cdf4abc8f26e2bae3 columnFamilyName f 2023-11-19T01:29:08,301 DEBUG [StoreOpener-4b19ffa2d6c4736cdf4abc8f26e2bae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:08,302 INFO [StoreOpener-4b19ffa2d6c4736cdf4abc8f26e2bae3-1 {}] regionserver.HStore(324): Store=4b19ffa2d6c4736cdf4abc8f26e2bae3/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:08,303 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns4/test-17003573419743/4b19ffa2d6c4736cdf4abc8f26e2bae3 2023-11-19T01:29:08,303 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns4/test-17003573419743/4b19ffa2d6c4736cdf4abc8f26e2bae3 2023-11-19T01:29:08,306 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 4b19ffa2d6c4736cdf4abc8f26e2bae3 2023-11-19T01:29:08,309 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns4/test-17003573419743/4b19ffa2d6c4736cdf4abc8f26e2bae3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:29:08,309 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 4b19ffa2d6c4736cdf4abc8f26e2bae3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11868202400, jitterRate=0.10531248152256012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:29:08,310 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 4b19ffa2d6c4736cdf4abc8f26e2bae3: 2023-11-19T01:29:08,311 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3., pid=24, masterSystemTime=1700357348291 2023-11-19T01:29:08,313 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. 2023-11-19T01:29:08,313 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. 2023-11-19T01:29:08,314 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=4b19ffa2d6c4736cdf4abc8f26e2bae3, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:08,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-11-19T01:29:08,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 4b19ffa2d6c4736cdf4abc8f26e2bae3, server=jenkins-hbase6.apache.org,40967,1700357333274 in 177 msec 2023-11-19T01:29:08,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-11-19T01:29:08,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-17003573419743, region=4b19ffa2d6c4736cdf4abc8f26e2bae3, ASSIGN in 336 msec 2023-11-19T01:29:08,321 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-17003573419743 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:29:08,321 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-17003573419743","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357348321"}]},"ts":"1700357348321"} 2023-11-19T01:29:08,323 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-17003573419743, state=ENABLED in hbase:meta 2023-11-19T01:29:08,326 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-17003573419743 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:29:08,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-17003573419743 in 787 msec 2023-11-19T01:29:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:29:08,650 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns4:test-17003573419743, procId: 22 completed 2023-11-19T01:29:08,650 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-17003573419743 get assigned. Timeout = 60000ms 2023-11-19T01:29:08,650 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:29:08,654 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-17003573419743 assigned to meta. Checking AM states. 2023-11-19T01:29:08,654 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:29:08,654 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-17003573419743 assigned. 2023-11-19T01:29:08,655 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x344d9ed9 to 127.0.0.1:59682 2023-11-19T01:29:08,655 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:08,696 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackupWithFailures#testIncBackupRestore[0] Thread=587, OpenFileDescriptor=696, MaxFileDescriptor=60000, SystemLoadAverage=124, ProcessCount=166, AvailableMemoryMB=3586 2023-11-19T01:29:08,696 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.ResourceChecker(130): Thread=587 is superior to 500 2023-11-19T01:29:08,699 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] backup.TestIncrementalBackupWithFailures(82): create full backup image for all tables 2023-11-19T01:29:08,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$14(2731): Client=jenkins//172.31.2.20 modify table test-1700357341974 from 'test-1700357341974', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1700357341974', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=test-1700357341974 2023-11-19T01:29:09,141 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/.tabledesc/.tableinfo.0000000001.367 2023-11-19T01:29:09,141 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/.tabledesc/.tableinfo.0000000002.598 2023-11-19T01:29:09,147 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=test-1700357341974}] 2023-11-19T01:29:09,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1700357341974, region=3c282db26b60e0d893a6e12adb946977, REOPEN/MOVE}] 2023-11-19T01:29:09,163 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1700357341974, region=3c282db26b60e0d893a6e12adb946977, REOPEN/MOVE 2023-11-19T01:29:09,164 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=3c282db26b60e0d893a6e12adb946977, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:09,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; CloseRegionProcedure 3c282db26b60e0d893a6e12adb946977, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:29:09,188 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:29:09,324 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:09,326 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(119): Close 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:09,327 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1668): Closing 3c282db26b60e0d893a6e12adb946977, disabling compactions & flushes 2023-11-19T01:29:09,327 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1690): Closing region test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:09,328 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:09,328 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1778): Acquired close lock on test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. after waiting 0 ms 2023-11-19T01:29:09,328 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1788): Updates disabled for region test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:09,328 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2819): Flushing 3c282db26b60e0d893a6e12adb946977 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-19T01:29:09,408 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/.tmp/f/cf30e45970ff42538eecb5e860161b86 is 37, key is row10/f:q1/1700357344624/Put/seqid=0 2023-11-19T01:29:09,827 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/.tmp/f/cf30e45970ff42538eecb5e860161b86 2023-11-19T01:29:09,899 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/.tmp/f/cf30e45970ff42538eecb5e860161b86 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f/cf30e45970ff42538eecb5e860161b86 2023-11-19T01:29:09,907 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f/cf30e45970ff42538eecb5e860161b86, entries=99, sequenceid=103, filesize=8.2 K 2023-11-19T01:29:09,910 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 3c282db26b60e0d893a6e12adb946977 in 582ms, sequenceid=103, compaction requested=false 2023-11-19T01:29:09,920 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-11-19T01:29:09,921 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:09,922 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1909): Closed test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:09,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1622): Region close journal for 3c282db26b60e0d893a6e12adb946977: 2023-11-19T01:29:09,922 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionServer(3735): Not adding moved region record: 3c282db26b60e0d893a6e12adb946977 to self. 2023-11-19T01:29:09,925 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(163): Closed 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:09,925 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=3c282db26b60e0d893a6e12adb946977, regionState=CLOSED 2023-11-19T01:29:09,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-11-19T01:29:09,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; CloseRegionProcedure 3c282db26b60e0d893a6e12adb946977, server=jenkins-hbase6.apache.org,40967,1700357333274 in 761 msec 2023-11-19T01:29:09,931 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1700357341974, region=3c282db26b60e0d893a6e12adb946977, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase6.apache.org,40967,1700357333274; forceNewPlan=false, retain=true 2023-11-19T01:29:10,081 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=3c282db26b60e0d893a6e12adb946977, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:10,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=29, ppid=27, state=RUNNABLE; OpenRegionProcedure 3c282db26b60e0d893a6e12adb946977, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:29:10,236 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:10,240 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(130): Open test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:10,240 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7254): Opening region: {ENCODED => 3c282db26b60e0d893a6e12adb946977, NAME => 'test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:29:10,241 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:29:10,241 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1700357341974 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:10,241 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(888): Instantiated test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:10,241 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7296): checking encryption for 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:10,241 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7299): checking classloading for 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:10,243 INFO [StoreOpener-3c282db26b60e0d893a6e12adb946977-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:10,245 INFO [StoreOpener-3c282db26b60e0d893a6e12adb946977-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c282db26b60e0d893a6e12adb946977 columnFamilyName f 2023-11-19T01:29:10,245 DEBUG [StoreOpener-3c282db26b60e0d893a6e12adb946977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:10,255 DEBUG [StoreOpener-3c282db26b60e0d893a6e12adb946977-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f/cf30e45970ff42538eecb5e860161b86 2023-11-19T01:29:10,255 INFO [StoreOpener-3c282db26b60e0d893a6e12adb946977-1 {}] regionserver.HStore(324): Store=3c282db26b60e0d893a6e12adb946977/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:10,255 INFO [StoreOpener-3c282db26b60e0d893a6e12adb946977-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:10,257 INFO [StoreOpener-3c282db26b60e0d893a6e12adb946977-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c282db26b60e0d893a6e12adb946977 columnFamilyName f3 2023-11-19T01:29:10,257 DEBUG [StoreOpener-3c282db26b60e0d893a6e12adb946977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:10,258 INFO [StoreOpener-3c282db26b60e0d893a6e12adb946977-1 {}] regionserver.HStore(324): Store=3c282db26b60e0d893a6e12adb946977/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:10,259 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:10,260 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:10,262 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1700357341974 descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:29:10,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1076): writing seq id for 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:10,266 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): Opened 3c282db26b60e0d893a6e12adb946977; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9629445440, jitterRate=-0.10318800806999207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:29:10,266 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(992): Region open journal for 3c282db26b60e0d893a6e12adb946977: 2023-11-19T01:29:10,267 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977., pid=29, masterSystemTime=1700357350236 2023-11-19T01:29:10,269 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:10,269 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(158): Opened test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:10,270 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=3c282db26b60e0d893a6e12adb946977, regionState=OPEN, openSeqNum=107, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:10,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=29, resume processing ppid=27 2023-11-19T01:29:10,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, ppid=27, state=SUCCESS; OpenRegionProcedure 3c282db26b60e0d893a6e12adb946977, server=jenkins-hbase6.apache.org,40967,1700357333274 in 189 msec 2023-11-19T01:29:10,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-11-19T01:29:10,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=test-1700357341974, region=3c282db26b60e0d893a6e12adb946977, REOPEN/MOVE in 1.1120 sec 2023-11-19T01:29:10,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=26, resume processing ppid=25 2023-11-19T01:29:10,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, ppid=25, state=SUCCESS; ReopenTableRegionsProcedure table=test-1700357341974 in 1.1320 sec 2023-11-19T01:29:10,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; ModifyTableProcedure table=test-1700357341974 in 1.5700 sec 2023-11-19T01:29:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-19T01:29:10,294 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x47a13bc1 to 127.0.0.1:59682 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:29:10,300 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed611f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:10,302 DEBUG [hconnection-0x28734e7e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:10,303 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:42042, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:10,335 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:10,336 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:36044, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:10,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'backup'} 2023-11-19T01:29:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=30, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-11-19T01:29:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=30 2023-11-19T01:29:10,356 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:29:10,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 14 msec 2023-11-19T01:29:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=30 2023-11-19T01:29:10,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-11-19T01:29:10,472 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:29:10,472 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:10,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 31 2023-11-19T01:29:10,473 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:29:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=31 2023-11-19T01:29:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=31 2023-11-19T01:29:10,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=31 2023-11-19T01:29:10,887 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa 2023-11-19T01:29:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=31 2023-11-19T01:29:11,300 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:11,300 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 25b6494b58e50dbfdbcd9050bce4992b, disabling compactions & flushes 2023-11-19T01:29:11,300 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:11,300 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:11,300 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. after waiting 0 ms 2023-11-19T01:29:11,300 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:11,300 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:11,300 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:29:11,302 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:29:11,302 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1700357351302"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357351302"}]},"ts":"1700357351302"} 2023-11-19T01:29:11,304 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:29:11,305 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:29:11,305 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357351305"}]},"ts":"1700357351305"} 2023-11-19T01:29:11,306 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-19T01:29:11,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN}] 2023-11-19T01:29:11,316 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN 2023-11-19T01:29:11,317 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,40967,1700357333274; forceNewPlan=false, retain=false 2023-11-19T01:29:11,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1700357341974 2023-11-19T01:29:11,341 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1700357341974 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:29:11,342 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17003573419741 2023-11-19T01:29:11,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17003573419741 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:29:11,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17003573419743 2023-11-19T01:29:11,343 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17003573419743 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:29:11,343 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17003573419742 2023-11-19T01:29:11,343 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17003573419742 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:29:11,469 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=32 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:11,471 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; OpenRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:29:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=31 2023-11-19T01:29:11,623 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:11,628 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(130): Open backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:11,628 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7254): Opening region: {ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:29:11,628 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:29:11,629 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:11,629 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(888): Instantiated backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:11,629 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7296): checking encryption for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:11,629 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7299): checking classloading for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:11,631 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:11,633 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25b6494b58e50dbfdbcd9050bce4992b columnFamilyName meta 2023-11-19T01:29:11,633 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:11,634 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(324): Store=25b6494b58e50dbfdbcd9050bce4992b/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:11,634 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:11,636 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25b6494b58e50dbfdbcd9050bce4992b columnFamilyName session 2023-11-19T01:29:11,636 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:11,636 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(324): Store=25b6494b58e50dbfdbcd9050bce4992b/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:11,637 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:11,638 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:11,639 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:29:11,641 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1076): writing seq id for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:11,645 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:29:11,646 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): Opened 25b6494b58e50dbfdbcd9050bce4992b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9742260000, jitterRate=-0.09268133342266083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:29:11,646 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(992): Region open journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:29:11,647 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b., pid=33, masterSystemTime=1700357351623 2023-11-19T01:29:11,649 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:11,649 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(158): Opened backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:11,650 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=32 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:11,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-11-19T01:29:11,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; OpenRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274 in 181 msec 2023-11-19T01:29:11,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=32, resume processing ppid=31 2023-11-19T01:29:11,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, ppid=31, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN in 340 msec 2023-11-19T01:29:11,657 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:29:11,657 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357351657"}]},"ts":"1700357351657"} 2023-11-19T01:29:11,659 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-19T01:29:11,662 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:29:11,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, state=SUCCESS; CreateTableProcedure table=backup:system in 1.1940 sec 2023-11-19T01:29:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=31 2023-11-19T01:29:12,581 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system, procId: 31 completed 2023-11-19T01:29:12,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-11-19T01:29:12,599 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:29:12,600 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:12,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 34 2023-11-19T01:29:12,601 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:29:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:29:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:29:12,837 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17003573419743' 2023-11-19T01:29:12,838 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-19T01:29:12,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:29:13,017 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2eb9f9fa54beda37677c07a7a5b64fe7, NAME => 'backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa 2023-11-19T01:29:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:29:13,429 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:13,429 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 2eb9f9fa54beda37677c07a7a5b64fe7, disabling compactions & flushes 2023-11-19T01:29:13,429 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. 2023-11-19T01:29:13,429 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. 2023-11-19T01:29:13,429 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. after waiting 0 ms 2023-11-19T01:29:13,429 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. 2023-11-19T01:29:13,429 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. 2023-11-19T01:29:13,429 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2eb9f9fa54beda37677c07a7a5b64fe7: 2023-11-19T01:29:13,431 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:29:13,431 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1700357353431"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357353431"}]},"ts":"1700357353431"} 2023-11-19T01:29:13,433 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:29:13,434 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:29:13,434 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357353434"}]},"ts":"1700357353434"} 2023-11-19T01:29:13,435 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-11-19T01:29:13,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=2eb9f9fa54beda37677c07a7a5b64fe7, ASSIGN}] 2023-11-19T01:29:13,445 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=2eb9f9fa54beda37677c07a7a5b64fe7, ASSIGN 2023-11-19T01:29:13,445 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=2eb9f9fa54beda37677c07a7a5b64fe7, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,40967,1700357333274; forceNewPlan=false, retain=false 2023-11-19T01:29:13,597 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=2eb9f9fa54beda37677c07a7a5b64fe7, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:13,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE; OpenRegionProcedure 2eb9f9fa54beda37677c07a7a5b64fe7, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:29:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:29:13,751 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:13,755 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. 2023-11-19T01:29:13,755 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7254): Opening region: {ENCODED => 2eb9f9fa54beda37677c07a7a5b64fe7, NAME => 'backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:29:13,756 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:29:13,756 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 2eb9f9fa54beda37677c07a7a5b64fe7 2023-11-19T01:29:13,756 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:13,756 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7296): checking encryption for 2eb9f9fa54beda37677c07a7a5b64fe7 2023-11-19T01:29:13,756 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7299): checking classloading for 2eb9f9fa54beda37677c07a7a5b64fe7 2023-11-19T01:29:13,758 INFO [StoreOpener-2eb9f9fa54beda37677c07a7a5b64fe7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 2eb9f9fa54beda37677c07a7a5b64fe7 2023-11-19T01:29:13,760 INFO [StoreOpener-2eb9f9fa54beda37677c07a7a5b64fe7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2eb9f9fa54beda37677c07a7a5b64fe7 columnFamilyName meta 2023-11-19T01:29:13,760 DEBUG [StoreOpener-2eb9f9fa54beda37677c07a7a5b64fe7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:13,760 INFO [StoreOpener-2eb9f9fa54beda37677c07a7a5b64fe7-1 {}] regionserver.HStore(324): Store=2eb9f9fa54beda37677c07a7a5b64fe7/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:13,760 INFO [StoreOpener-2eb9f9fa54beda37677c07a7a5b64fe7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 2eb9f9fa54beda37677c07a7a5b64fe7 2023-11-19T01:29:13,762 INFO [StoreOpener-2eb9f9fa54beda37677c07a7a5b64fe7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2eb9f9fa54beda37677c07a7a5b64fe7 columnFamilyName session 2023-11-19T01:29:13,762 DEBUG [StoreOpener-2eb9f9fa54beda37677c07a7a5b64fe7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:13,763 INFO [StoreOpener-2eb9f9fa54beda37677c07a7a5b64fe7-1 {}] regionserver.HStore(324): Store=2eb9f9fa54beda37677c07a7a5b64fe7/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:13,764 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system_bulk/2eb9f9fa54beda37677c07a7a5b64fe7 2023-11-19T01:29:13,764 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system_bulk/2eb9f9fa54beda37677c07a7a5b64fe7 2023-11-19T01:29:13,766 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:29:13,768 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1076): writing seq id for 2eb9f9fa54beda37677c07a7a5b64fe7 2023-11-19T01:29:13,771 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system_bulk/2eb9f9fa54beda37677c07a7a5b64fe7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:29:13,772 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): Opened 2eb9f9fa54beda37677c07a7a5b64fe7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11398284640, jitterRate=0.06154797971248627}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:29:13,772 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(992): Region open journal for 2eb9f9fa54beda37677c07a7a5b64fe7: 2023-11-19T01:29:13,773 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7., pid=36, masterSystemTime=1700357353751 2023-11-19T01:29:13,775 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. 2023-11-19T01:29:13,775 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. 2023-11-19T01:29:13,776 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=2eb9f9fa54beda37677c07a7a5b64fe7, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:13,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=36, resume processing ppid=35 2023-11-19T01:29:13,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, ppid=35, state=SUCCESS; OpenRegionProcedure 2eb9f9fa54beda37677c07a7a5b64fe7, server=jenkins-hbase6.apache.org,40967,1700357333274 in 179 msec 2023-11-19T01:29:13,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-11-19T01:29:13,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=2eb9f9fa54beda37677c07a7a5b64fe7, ASSIGN in 338 msec 2023-11-19T01:29:13,784 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:29:13,784 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357353784"}]},"ts":"1700357353784"} 2023-11-19T01:29:13,786 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-11-19T01:29:13,789 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:29:13,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.1930 sec 2023-11-19T01:29:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:29:14,709 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system_bulk, procId: 34 completed 2023-11-19T01:29:14,720 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:29:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40967 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2023-11-19T01:29:14,755 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357354755 (current time:1700357354755). 2023-11-19T01:29:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:29:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x059b117e to 127.0.0.1:59682 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:29:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47018248, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:14,769 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:42048, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x059b117e to 127.0.0.1:59682 2023-11-19T01:29:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=37, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-11-19T01:29:14,786 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-19T01:29:14,791 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:14,803 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot, current working dir path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:14,805 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-19T01:29:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-19T01:29:15,218 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:29:15,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b}] 2023-11-19T01:29:15,232 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-19T01:29:15,391 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:15,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40967 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=38 2023-11-19T01:29:15,393 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:15,395 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(2819): Flushing 25b6494b58e50dbfdbcd9050bce4992b 2/2 column families, dataSize=45 B heapSize=632 B 2023-11-19T01:29:15,410 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/1b3b4537222a44d3a2ca425154b42b44 is 49, key is activesession:/session:c/1700357354741/Put/seqid=0 2023-11-19T01:29:15,425 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:29:15,820 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/1b3b4537222a44d3a2ca425154b42b44 2023-11-19T01:29:15,830 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/1b3b4537222a44d3a2ca425154b42b44 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44 2023-11-19T01:29:15,840 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44, entries=1, sequenceid=5, filesize=4.9 K 2023-11-19T01:29:15,842 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 25b6494b58e50dbfdbcd9050bce4992b in 447ms, sequenceid=5, compaction requested=false 2023-11-19T01:29:15,843 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(2520): Flush status journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:29:15,843 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. for snapshot_backup_system completed. 2023-11-19T01:29:15,845 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:29:15,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:29:15,861 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-19T01:29:15,861 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44] hfiles 2023-11-19T01:29:15,861 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44 for snapshot=snapshot_backup_system 2023-11-19T01:29:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-19T01:29:16,279 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:16,283 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=38 2023-11-19T01:29:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster(4026): Remote procedure done, pid=38 2023-11-19T01:29:16,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:16,289 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:16,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=38, resume processing ppid=37 2023-11-19T01:29:16,292 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:29:16,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, ppid=37, state=SUCCESS; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b in 1.0680 sec 2023-11-19T01:29:16,294 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:16,297 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:16,297 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:29:16,300 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:29:16,721 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:16,734 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:16,735 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:29:16,737 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:16,737 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-11-19T01:29:16,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.9580 sec 2023-11-19T01:29:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2023-11-19T01:29:16,894 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 37 completed 2023-11-19T01:29:16,895 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(115): Backup backup_1700357350318 started at 1700357356894. 2023-11-19T01:29:16,916 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(119): Backup session backup_1700357350318 has been started. 2023-11-19T01:29:16,919 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:29:16,929 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:29:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:29:16,933 INFO [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:29:16,933 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:29:16,934 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:29:16,934 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:29:16,935 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:16,935 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:16,938 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-19T01:29:16,938 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:16,938 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:29:16,938 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:29:16,938 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:16,938 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:29:16,939 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:16,939 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:16,940 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:29:16,940 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:16,940 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:29:16,941 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:29:16,942 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:29:16,942 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:29:16,942 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:29:16,943 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:29:16,943 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:29:16,943 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,40967,1700357333274' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:29:16,945 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:16,945 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:16,945 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:16,945 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:16,945 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:16,946 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-19T01:29:16,946 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:29:16,946 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:16,946 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:16,947 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:16,947 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:16,947 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:16,948 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,40967,1700357333274' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:29:16,948 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@565abeab[Count = 0] remaining members to acquire global barrier 2023-11-19T01:29:16,948 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:29:16,948 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:16,950 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:16,950 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:16,950 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:16,950 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:29:16,950 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:16,950 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:29:16,951 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:29:16,951 DEBUG [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:16,952 INFO [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357335002 highest: 1700357337536 on jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:16,952 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0:(num 1700357335002) roll requested 2023-11-19T01:29:16,965 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952, exclude list is [], retry=0 2023-11-19T01:29:16,969 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:29:16,972 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357335002 with entries=18, filesize=3.96 KB; new WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 2023-11-19T01:29:16,973 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:29:16,973 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357335002 is not closed yet, will try archiving it next time 2023-11-19T01:29:16,973 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta:.meta(num 1700357336010) roll requested 2023-11-19T01:29:16,985 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357356973.meta, exclude list is [], retry=0 2023-11-19T01:29:16,989 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:29:16,991 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357336010.meta with entries=40, filesize=12.94 KB; new WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357356973.meta 2023-11-19T01:29:16,991 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:29:16,991 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357336010.meta is not closed yet, will try archiving it next time 2023-11-19T01:29:16,991 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1:(num 1700357337536) roll requested 2023-11-19T01:29:17,003 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991, exclude list is [], retry=0 2023-11-19T01:29:17,006 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:29:17,008 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357337536 with entries=11, filesize=2.04 KB; new WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 2023-11-19T01:29:17,008 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:29:17,008 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357337536 is not closed yet, will try archiving it next time 2023-11-19T01:29:17,013 DEBUG [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-11-19T01:29:17,013 INFO [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357356952 on jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:17,013 DEBUG [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:17,014 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:36054, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:29:17,032 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:29:17,032 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,40967,1700357333274' in zk 2023-11-19T01:29:17,034 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:17,034 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:29:17,035 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:29:17,035 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:29:17,035 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:17,035 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:17,035 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:17,036 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:17,036 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:17,036 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:17,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:17,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:17,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:17,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:17,039 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,40967,1700357333274': 2023-11-19T01:29:17,039 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,40967,1700357333274' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:29:17,039 INFO [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:29:17,039 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:29:17,039 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:29:17,039 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:29:17,039 INFO [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:29:17,043 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:29:17,043 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:29:17,043 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-19T01:29:17,043 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:29:17,043 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:17,043 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:17,043 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:17,043 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:17,043 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:29:17,043 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:29:17,043 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:17,044 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:17,044 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:17,044 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:17,044 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:17,045 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:17,045 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:17,045 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:17,046 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:17,046 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:17,047 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:17,055 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:17,055 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:17,055 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:17,055 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-19T01:29:17,055 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:17,055 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:17,055 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:29:17,055 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:17,055 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40967-0x10123a5ddbf0001, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-19T01:29:17,055 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:29:17,055 DEBUG [Listener at localhost/35467-EventThread {}] zookeeper.ZKWatcher(604): master:36603-0x10123a5ddbf0000, quorum=127.0.0.1:59682, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:29:17,055 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:29:17,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:29:17,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:29:17,056 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:29:17,056 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:29:17,058 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-19T01:29:17,058 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-19T01:29:17,158 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-19T01:29:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:29:17,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 } 2023-11-19T01:29:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357357168 (current time:1700357357168). 2023-11-19T01:29:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357357167_default_test-1700357341974 VERSION not specified, setting to 2 2023-11-19T01:29:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x08b4d92c to 127.0.0.1:59682 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:29:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2495d908, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:17,177 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:42050, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x08b4d92c to 127.0.0.1:59682 2023-11-19T01:29:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 } 2023-11-19T01:29:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 }, snapshot procedure id = 39 2023-11-19T01:29:17,181 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-19T01:29:17,182 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:17,185 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot, current working dir path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_1700357357167_default_test-1700357341974 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:17,187 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-19T01:29:17,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-19T01:29:17,597 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:29:17,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 3c282db26b60e0d893a6e12adb946977}] 2023-11-19T01:29:17,600 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:17,751 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:17,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40967 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2023-11-19T01:29:17,752 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:17,753 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2819): Flushing 3c282db26b60e0d893a6e12adb946977 2/2 column families, dataSize=324 B heapSize=1.25 KB 2023-11-19T01:29:17,770 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/.tmp/f3/2b8a7ce7d1d04b6794a22e44f2c409fd is 58, key is row-test-1700357341974-3-0/f3:q1/1700357350307/Put/seqid=0 2023-11-19T01:29:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-19T01:29:18,178 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/.tmp/f3/2b8a7ce7d1d04b6794a22e44f2c409fd 2023-11-19T01:29:18,187 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/.tmp/f3/2b8a7ce7d1d04b6794a22e44f2c409fd as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f3/2b8a7ce7d1d04b6794a22e44f2c409fd 2023-11-19T01:29:18,194 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f3/2b8a7ce7d1d04b6794a22e44f2c409fd, entries=6, sequenceid=115, filesize=5.2 K 2023-11-19T01:29:18,196 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3022): Finished flush of dataSize ~324 B/324, heapSize ~1008 B/1008, currentSize=0 B/0 for 3c282db26b60e0d893a6e12adb946977 in 443ms, sequenceid=115, compaction requested=false 2023-11-19T01:29:18,196 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2520): Flush status journal for 3c282db26b60e0d893a6e12adb946977: 2023-11-19T01:29:18,196 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. for snapshot_1700357357167_default_test-1700357341974 completed. 2023-11-19T01:29:18,196 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(240): Storing 'test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977.' region-info for snapshot=snapshot_1700357357167_default_test-1700357341974 2023-11-19T01:29:18,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:29:18,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f/cf30e45970ff42538eecb5e860161b86] hfiles 2023-11-19T01:29:18,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f/cf30e45970ff42538eecb5e860161b86 for snapshot=snapshot_1700357357167_default_test-1700357341974 2023-11-19T01:29:18,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f3/2b8a7ce7d1d04b6794a22e44f2c409fd] hfiles 2023-11-19T01:29:18,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f3/2b8a7ce7d1d04b6794a22e44f2c409fd for snapshot=snapshot_1700357357167_default_test-1700357341974 2023-11-19T01:29:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-19T01:29:18,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:29:18,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=40 2023-11-19T01:29:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster(4026): Remote procedure done, pid=40 2023-11-19T01:29:18,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357357167_default_test-1700357341974 on region 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:18,608 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:29:18,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-11-19T01:29:18,611 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:29:18,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; SnapshotRegionProcedure 3c282db26b60e0d893a6e12adb946977 in 1.0110 sec 2023-11-19T01:29:18,612 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:18,613 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:18,613 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357357167_default_test-1700357341974 2023-11-19T01:29:18,614 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_1700357357167_default_test-1700357341974 2023-11-19T01:29:18,839 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-11-19T01:29:19,034 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:19,042 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:19,043 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_1700357357167_default_test-1700357341974 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/snapshot_1700357357167_default_test-1700357341974 2023-11-19T01:29:19,044 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:19,044 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 }, snapshot procedure id = 39 2023-11-19T01:29:19,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1700357357167_default_test-1700357341974 table=test-1700357341974 type=FLUSH ttl=0 } in 1.8650 sec 2023-11-19T01:29:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=39 2023-11-19T01:29:19,288 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: default:test-1700357341974, procId: 39 completed 2023-11-19T01:29:19,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 } 2023-11-19T01:29:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357359289 (current time:1700357359289). 2023-11-19T01:29:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357359289_ns2_test-17003573419741 VERSION not specified, setting to 2 2023-11-19T01:29:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2e185976 to 127.0.0.1:59682 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:29:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9820, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:19,304 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:42056, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2e185976 to 127.0.0.1:59682 2023-11-19T01:29:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:19,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=41, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 } 2023-11-19T01:29:19,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-11-19T01:29:19,309 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:19,310 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:19,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-19T01:29:19,314 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot, current working dir path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_1700357359289_ns2_test-17003573419741 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:19,317 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-19T01:29:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-19T01:29:19,727 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:29:19,727 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure 4626e266aa32d68d09dea77ed5ea495c}] 2023-11-19T01:29:19,729 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure 4626e266aa32d68d09dea77ed5ea495c 2023-11-19T01:29:19,880 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:19,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40967 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=42 2023-11-19T01:29:19,881 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:29:19,882 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(2819): Flushing 4626e266aa32d68d09dea77ed5ea495c 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-19T01:29:19,899 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/.tmp/f/95ebb5a7ae364a5f809d27737b76f4c8 is 37, key is row10/f:q1/1700357345331/Put/seqid=0 2023-11-19T01:29:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-19T01:29:20,307 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/.tmp/f/95ebb5a7ae364a5f809d27737b76f4c8 2023-11-19T01:29:20,317 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/.tmp/f/95ebb5a7ae364a5f809d27737b76f4c8 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/f/95ebb5a7ae364a5f809d27737b76f4c8 2023-11-19T01:29:20,324 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/f/95ebb5a7ae364a5f809d27737b76f4c8, entries=99, sequenceid=103, filesize=8.2 K 2023-11-19T01:29:20,325 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 4626e266aa32d68d09dea77ed5ea495c in 443ms, sequenceid=103, compaction requested=false 2023-11-19T01:29:20,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(2520): Flush status journal for 4626e266aa32d68d09dea77ed5ea495c: 2023-11-19T01:29:20,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. for snapshot_1700357359289_ns2_test-17003573419741 completed. 2023-11-19T01:29:20,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(240): Storing 'ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c.' region-info for snapshot=snapshot_1700357359289_ns2_test-17003573419741 2023-11-19T01:29:20,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:29:20,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/f/95ebb5a7ae364a5f809d27737b76f4c8] hfiles 2023-11-19T01:29:20,326 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/f/95ebb5a7ae364a5f809d27737b76f4c8 for snapshot=snapshot_1700357359289_ns2_test-17003573419741 2023-11-19T01:29:20,335 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:29:20,335 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=42 2023-11-19T01:29:20,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster(4026): Remote procedure done, pid=42 2023-11-19T01:29:20,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357359289_ns2_test-17003573419741 on region 4626e266aa32d68d09dea77ed5ea495c 2023-11-19T01:29:20,336 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure 4626e266aa32d68d09dea77ed5ea495c 2023-11-19T01:29:20,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=42, resume processing ppid=41 2023-11-19T01:29:20,339 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:29:20,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, ppid=41, state=SUCCESS; SnapshotRegionProcedure 4626e266aa32d68d09dea77ed5ea495c in 610 msec 2023-11-19T01:29:20,339 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:20,340 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:20,340 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357359289_ns2_test-17003573419741 2023-11-19T01:29:20,341 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_1700357359289_ns2_test-17003573419741 2023-11-19T01:29:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-19T01:29:20,757 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:20,764 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:20,764 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_1700357359289_ns2_test-17003573419741 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/snapshot_1700357359289_ns2_test-17003573419741 2023-11-19T01:29:20,766 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:20,766 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-11-19T01:29:20,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1700357359289_ns2_test-17003573419741 table=ns2:test-17003573419741 type=FLUSH ttl=0 } in 1.4600 sec 2023-11-19T01:29:21,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-19T01:29:21,341 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-11-19T01:29:21,342 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-11-19T01:29:21,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-11-19T01:29:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-19T01:29:21,418 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: ns2:test-17003573419741, procId: 41 completed 2023-11-19T01:29:21,419 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357350318 2023-11-19T01:29:21,419 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:29:21,420 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-19T01:29:21,423 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357350318_test-1700357341974 2023-11-19T01:29:21,423 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357357167_default_test-1700357341974 to hdfs://localhost:39587/backupUT/backup_1700357350318/default/test-1700357341974/ 2023-11-19T01:29:21,433 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:29:21,464 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39587, inputRoot=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa 2023-11-19T01:29:21,464 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,36603,1700357330230_1568903676_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39587/backupUT/backup_1700357350318/default/test-1700357341974, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39587/backupUT/backup_1700357350318/default/test-1700357341974/.hbase-snapshot/.tmp/snapshot_1700357357167_default_test-1700357341974 2023-11-19T01:29:21,464 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39587, inputRoot=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa, snapshotDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/snapshot_1700357357167_default_test-1700357341974. 2023-11-19T01:29:21,478 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/snapshot_1700357357167_default_test-1700357341974 to hdfs://localhost:39587/backupUT/backup_1700357350318/default/test-1700357341974/.hbase-snapshot/.tmp/snapshot_1700357357167_default_test-1700357341974 2023-11-19T01:29:21,915 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:21,915 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:21,917 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:21,917 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:22,978 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop-7583597720408680213.jar 2023-11-19T01:29:22,979 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:22,979 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:22,979 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:22,980 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:22,980 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:22,981 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:22,981 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:22,982 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:22,982 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:22,983 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:22,984 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:29:22,984 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:29:22,985 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:29:22,985 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:29:22,986 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:29:22,986 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:29:22,986 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:29:22,987 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:29:22,987 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:29:22,988 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:29:22,988 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:29:22,992 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:22,993 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:22,993 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:22,994 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:22,994 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:22,995 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:22,995 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:23,055 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-19T01:29:23,079 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357357167_default_test-1700357341974' hfile list 2023-11-19T01:29:23,091 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(674): export split=0 size=13.4 K 2023-11-19T01:29:23,317 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.local.dir/1700357363269/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:29:23,317 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.local.dir/1700357363269/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:29:23,476 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:29:23,505 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f/cf30e45970ff42538eecb5e860161b86 output=hdfs://localhost:39587/backupUT/backup_1700357350318/default/test-1700357341974/archive/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f/cf30e45970ff42538eecb5e860161b86 2023-11-19T01:29:23,505 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-11-19T01:29:23,940 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f3/2b8a7ce7d1d04b6794a22e44f2c409fd output=hdfs://localhost:39587/backupUT/backup_1700357350318/default/test-1700357341974/archive/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f3/2b8a7ce7d1d04b6794a22e44f2c409fd 2023-11-19T01:29:23,940 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=5370 (5.2 K) time=0sec 5.121M/sec 2023-11-19T01:29:24,376 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:29:24,378 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:29:24,386 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357357167_default_test-1700357341974 2023-11-19T01:29:24,386 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357357167_default_test-1700357341974 finished. 2023-11-19T01:29:24,386 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357350318_ns2:test-17003573419741 2023-11-19T01:29:24,387 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357359289_ns2_test-17003573419741 to hdfs://localhost:39587/backupUT/backup_1700357350318/ns2/test-17003573419741/ 2023-11-19T01:29:24,387 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:29:24,413 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:39587, inputRoot=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa 2023-11-19T01:29:24,413 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,36603,1700357330230_-860210360_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:39587/backupUT/backup_1700357350318/ns2/test-17003573419741, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:39587/backupUT/backup_1700357350318/ns2/test-17003573419741/.hbase-snapshot/.tmp/snapshot_1700357359289_ns2_test-17003573419741 2023-11-19T01:29:24,413 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:39587, inputRoot=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa, snapshotDir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/snapshot_1700357359289_ns2_test-17003573419741. 2023-11-19T01:29:24,422 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/snapshot_1700357359289_ns2_test-17003573419741 to hdfs://localhost:39587/backupUT/backup_1700357350318/ns2/test-17003573419741/.hbase-snapshot/.tmp/snapshot_1700357359289_ns2_test-17003573419741 2023-11-19T01:29:24,839 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:24,839 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:24,840 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:24,841 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:25,841 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop-2003331453829974331.jar 2023-11-19T01:29:25,842 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:25,842 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:25,843 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:25,843 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:25,843 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:25,844 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:25,844 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:25,845 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:25,845 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:25,846 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:25,847 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:29:25,847 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:29:25,847 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:29:25,848 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:29:25,849 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:29:25,849 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:29:25,849 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:29:25,850 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:29:25,850 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:29:25,851 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:29:25,851 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:29:25,852 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:25,852 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:25,853 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:25,854 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:25,854 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:25,855 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:25,855 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:25,882 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-19T01:29:25,900 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357359289_ns2_test-17003573419741' hfile list 2023-11-19T01:29:25,903 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-19T01:29:25,979 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.local.dir/1700357365945/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:29:25,979 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.local.dir/1700357365945/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:29:26,035 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:29:26,062 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/f/95ebb5a7ae364a5f809d27737b76f4c8 output=hdfs://localhost:39587/backupUT/backup_1700357350318/ns2/test-17003573419741/archive/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/f/95ebb5a7ae364a5f809d27737b76f4c8 2023-11-19T01:29:26,062 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-11-19T01:29:27,016 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:29:27,017 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:29:27,024 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357359289_ns2_test-17003573419741 2023-11-19T01:29:27,024 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357359289_ns2_test-17003573419741 finished. 2023-11-19T01:29:27,025 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1034): test-1700357341974 2023-11-19T01:29:27,025 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1034): ns2:test-17003573419741 2023-11-19T01:29:27,122 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357350318 2023-11-19T01:29:27,122 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:29:27,534 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39587/backupUT/backup_1700357350318/.backup.manifest 2023-11-19T01:29:27,534 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357350318 2023-11-19T01:29:27,534 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:29:27,940 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:39587/backupUT/backup_1700357350318/.backup.manifest 2023-11-19T01:29:27,940 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(358): Backup backup_1700357350318 finished: type=FULL,tablelist=test-1700357341974;ns2:test-17003573419741,targetRootDir=hdfs://localhost:39587/backupUT,startts=1700357356894,completets=1700357367119,bytescopied=0 2023-11-19T01:29:27,940 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:29:27,940 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357357167_default_test-1700357341974 2023-11-19T01:29:27,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357357167_default_test-1700357341974" 2023-11-19T01:29:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357357167_default_test-1700357341974 2023-11-19T01:29:27,952 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357357167_default_test-1700357341974 for backup backup_1700357350318 succeeded. 2023-11-19T01:29:27,952 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357359289_ns2_test-17003573419741 2023-11-19T01:29:27,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357359289_ns2_test-17003573419741" 2023-11-19T01:29:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357359289_ns2_test-17003573419741 2023-11-19T01:29:27,956 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357359289_ns2_test-17003573419741 for backup backup_1700357350318 succeeded. 2023-11-19T01:29:27,957 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:29:27,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:29:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:29:27,972 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:29:27,979 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:29:27,996 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(376): Backup backup_1700357350318 completed. 2023-11-19T01:29:28,127 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] backup.TestIncrementalBackupWithFailures(104): writing 99 rows to test-1700357341974 2023-11-19T01:29:28,145 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] backup.TestIncrementalBackupWithFailures(108): written 99 rows to test-1700357341974 2023-11-19T01:29:28,160 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] backup.TestIncrementalBackupWithFailures(120): written 5 rows to ns2:test-17003573419741 2023-11-19T01:29:28,161 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 0 2023-11-19T01:29:28,199 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@172e8bf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:28,200 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:28,204 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:46026, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:28,208 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:28,209 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37504, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:28,264 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:29:28,268 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_0 in testing java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:123) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:29:28,268 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(240): BackupId=backup_1700357368236,startts=0,failedts=1700357368268,failedphase=REQUEST,failedmessage=Failed stage stage_0 in testing 2023-11-19T01:29:28,268 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-19T01:29:28,270 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1550): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2023-11-19T01:29:28,270 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:29:28,272 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1579): Snapshot snapshot_backup_system does not exists 2023-11-19T01:29:28,273 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-11-19T01:29:28,274 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:29:28,276 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(246): Backup backup_1700357368236 failed. Backup session finished. Status: FAILURE 2023-11-19T01:29:28,276 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:28,277 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:123) ~[test-classes/:?] ... 41 more 2023-11-19T01:29:28,281 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 1 2023-11-19T01:29:28,299 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677ce17b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:28,300 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:28,301 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:46028, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:28,304 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:28,306 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37520, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:28,355 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:29:28,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357368358 (current time:1700357368358). 2023-11-19T01:29:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:29:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@575b31e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:28,368 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37534, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=43, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-11-19T01:29:28,372 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-19T01:29:28,373 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:28,377 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot, current working dir path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:28,380 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:28,387 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:29:28,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b}] 2023-11-19T01:29:28,389 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-19T01:29:28,484 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-19T01:29:28,486 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:46044, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-19T01:29:28,540 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:28,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40967 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=44 2023-11-19T01:29:28,540 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:28,541 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2819): Flushing 25b6494b58e50dbfdbcd9050bce4992b 2/2 column families, dataSize=1.83 KB heapSize=3.35 KB 2023-11-19T01:29:28,556 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 is 171, key is trslm:hdfs://localhost:39587/backupUT\x00ns2:test-17003573419741/meta:log-roll-map/1700357367106/Put/seqid=0 2023-11-19T01:29:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-19T01:29:28,961 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785 B at sequenceid=20 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 2023-11-19T01:29:28,972 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/d9d812f8f01b44aa909e0990967aeae9 is 459, key is session:backup_1700357350318/session:context/1700357367978/Put/seqid=0 2023-11-19T01:29:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-19T01:29:28,977 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.07 KB at sequenceid=20 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/d9d812f8f01b44aa909e0990967aeae9 2023-11-19T01:29:28,984 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 2023-11-19T01:29:28,991 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1, entries=6, sequenceid=20, filesize=5.8 K 2023-11-19T01:29:28,992 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/d9d812f8f01b44aa909e0990967aeae9 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9 2023-11-19T01:29:28,998 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9, entries=3, sequenceid=20, filesize=5.7 K 2023-11-19T01:29:28,999 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(3022): Finished flush of dataSize ~1.83 KB/1879, heapSize ~3.32 KB/3400, currentSize=0 B/0 for 25b6494b58e50dbfdbcd9050bce4992b in 459ms, sequenceid=20, compaction requested=false 2023-11-19T01:29:28,999 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2520): Flush status journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:29:28,999 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. for snapshot_backup_system completed. 2023-11-19T01:29:29,000 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:29:29,000 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:29:29,000 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1] hfiles 2023-11-19T01:29:29,000 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 for snapshot=snapshot_backup_system 2023-11-19T01:29:29,000 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9] hfiles 2023-11-19T01:29:29,000 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44 for snapshot=snapshot_backup_system 2023-11-19T01:29:29,000 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9 for snapshot=snapshot_backup_system 2023-11-19T01:29:29,005 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:29,005 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=44 2023-11-19T01:29:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster(4026): Remote procedure done, pid=44 2023-11-19T01:29:29,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:29,007 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:29,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=44, resume processing ppid=43 2023-11-19T01:29:29,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, ppid=43, state=SUCCESS; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b in 621 msec 2023-11-19T01:29:29,009 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:29:29,010 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:29,011 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:29,011 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:29:29,015 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:29:29,425 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:29,433 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:29,434 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:29:29,438 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:29,438 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-11-19T01:29:29,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.0690 sec 2023-11-19T01:29:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=43 2023-11-19T01:29:29,477 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(115): Backup backup_1700357368330 started at 1700357369477. 2023-11-19T01:29:29,479 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(119): Backup session backup_1700357368330 has been started. 2023-11-19T01:29:29,479 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_1 in testing java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:29:29,480 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(240): BackupId=backup_1700357368330,startts=1700357369477,failedts=1700357369480,failedphase=REQUEST,failedmessage=Failed stage stage_1 in testing 2023-11-19T01:29:29,480 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-19T01:29:29,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable backup:system 2023-11-19T01:29:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=45, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-19T01:29:29,504 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357369504"}]},"ts":"1700357369504"} 2023-11-19T01:29:29,505 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-19T01:29:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-19T01:29:29,508 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-19T01:29:29,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, UNASSIGN}] 2023-11-19T01:29:29,510 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, UNASSIGN 2023-11-19T01:29:29,511 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:29,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; CloseRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:29:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-19T01:29:29,664 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:29,664 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(119): Close 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:29,664 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1668): Closing 25b6494b58e50dbfdbcd9050bce4992b, disabling compactions & flushes 2023-11-19T01:29:29,664 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1690): Closing region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:29,664 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:29,664 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. after waiting 0 ms 2023-11-19T01:29:29,664 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:29,664 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(2819): Flushing 25b6494b58e50dbfdbcd9050bce4992b 2/2 column families, dataSize=214 B heapSize=800 B 2023-11-19T01:29:29,668 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/11321f9b1d41400a976816ac18313d90 is 218, key is session:backup_1700357368330/session:context/1700357369478/Put/seqid=0 2023-11-19T01:29:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-19T01:29:30,073 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/11321f9b1d41400a976816ac18313d90 2023-11-19T01:29:30,081 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/11321f9b1d41400a976816ac18313d90 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/11321f9b1d41400a976816ac18313d90 2023-11-19T01:29:30,087 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/11321f9b1d41400a976816ac18313d90, entries=1, sequenceid=24, filesize=5.2 K 2023-11-19T01:29:30,088 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(3022): Finished flush of dataSize ~214 B/214, heapSize ~528 B/528, currentSize=0 B/0 for 25b6494b58e50dbfdbcd9050bce4992b in 424ms, sequenceid=24, compaction requested=true 2023-11-19T01:29:30,098 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1 2023-11-19T01:29:30,099 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:30,099 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1909): Closed backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:30,099 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1622): Region close journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:29:30,101 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(163): Closed 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:30,102 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=46 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=CLOSED 2023-11-19T01:29:30,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=47, resume processing ppid=46 2023-11-19T01:29:30,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, ppid=46, state=SUCCESS; CloseRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274 in 591 msec 2023-11-19T01:29:30,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-11-19T01:29:30,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, UNASSIGN in 596 msec 2023-11-19T01:29:30,108 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357370108"}]},"ts":"1700357370108"} 2023-11-19T01:29:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-19T01:29:30,110 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-19T01:29:30,113 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-19T01:29:30,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, state=SUCCESS; DisableTableProcedure table=backup:system in 618 msec 2023-11-19T01:29:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=45 2023-11-19T01:29:30,620 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357370620 (current time:1700357370620). 2023-11-19T01:29:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1700357370619 VERSION not specified, setting to 2 2023-11-19T01:29:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fb58ffb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:30,629 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52380, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-11-19T01:29:30,634 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-19T01:29:30,635 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:30,638 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot, current working dir path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357370619 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:30,641 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-19T01:29:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-19T01:29:31,049 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-19T01:29:31,054 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-19T01:29:31,054 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-19T01:29:31,054 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:31,056 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1] hfiles 2023-11-19T01:29:31,056 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 2023-11-19T01:29:31,056 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:31,058 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/11321f9b1d41400a976816ac18313d90, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9] hfiles 2023-11-19T01:29:31,058 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/11321f9b1d41400a976816ac18313d90 2023-11-19T01:29:31,058 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44 2023-11-19T01:29:31,058 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9 2023-11-19T01:29:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-19T01:29:31,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-19T01:29:31,471 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-19T01:29:31,472 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:31,473 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:31,473 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1700357370619 2023-11-19T01:29:31,474 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357370619 2023-11-19T01:29:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-19T01:29:31,887 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:31,896 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:31,896 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357370619 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1700357370619 2023-11-19T01:29:31,898 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=48, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:31,898 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 48 2023-11-19T01:29:31,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=48, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357370619 table=backup:system type=FLUSH ttl=0 } in 1.2670 sec 2023-11-19T01:29:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2023-11-19T01:29:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=49, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357368358 type: FLUSH version: 2 ttl: 0 ) 2023-11-19T01:29:32,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-19T01:29:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-19T01:29:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-19T01:29:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-19T01:29:33,183 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-11-19T01:29:33,183 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-11-19T01:29:33,185 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:33,191 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357368358 type: FLUSH version: 2 ttl: 0 2023-11-19T01:29:33,192 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system 2023-11-19T01:29:33,195 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-19T01:29:33,195 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:33,201 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:33,210 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/11321f9b1d41400a976816ac18313d90 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/archive/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/11321f9b1d41400a976816ac18313d90 2023-11-19T01:29:33,211 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:33,211 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357368358 type: FLUSH version: 2 ttl: 0 2023-11-19T01:29:33,212 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357373212"}]},"ts":"9223372036854775807"} 2023-11-19T01:29:33,215 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:29:33,215 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:29:33,215 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1700357373213"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357373213"}]},"ts":"1700357373213"} 2023-11-19T01:29:33,217 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:29:33,217 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-19T01:29:33,217 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:29:33,217 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-19T01:29:33,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357368358 type: FLUSH version: 2 ttl: 0 ) in 461 msec 2023-11-19T01:29:33,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=49 2023-11-19T01:29:33,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "hbase-failsafe-snapshot_backup_system-1700357370619" 2023-11-19T01:29:33,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1700357370619 2023-11-19T01:29:33,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$12(2662): Client=jenkins//172.31.2.20 enable backup:system 2023-11-19T01:29:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-19T01:29:33,397 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-19T01:29:33,398 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357373397"}]},"ts":"1700357373397"} 2023-11-19T01:29:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-19T01:29:33,399 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-19T01:29:33,407 DEBUG [PEWorker-5 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-19T01:29:33,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN}] 2023-11-19T01:29:33,409 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN 2023-11-19T01:29:33,409 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=51, ppid=50, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-19T01:29:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-19T01:29:33,561 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:33,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE; OpenRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:29:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-19T01:29:33,714 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:33,717 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(130): Open backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:33,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7254): Opening region: {ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:29:33,718 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:29:33,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:33,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(888): Instantiated backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:33,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7296): checking encryption for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:33,718 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7299): checking classloading for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:33,720 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:33,721 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25b6494b58e50dbfdbcd9050bce4992b columnFamilyName meta 2023-11-19T01:29:33,721 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:33,727 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 2023-11-19T01:29:33,727 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(324): Store=25b6494b58e50dbfdbcd9050bce4992b/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:33,727 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:33,728 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25b6494b58e50dbfdbcd9050bce4992b columnFamilyName session 2023-11-19T01:29:33,728 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:33,734 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44 2023-11-19T01:29:33,739 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9 2023-11-19T01:29:33,739 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(324): Store=25b6494b58e50dbfdbcd9050bce4992b/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:33,740 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:33,741 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:33,742 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:29:33,744 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1076): writing seq id for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:33,745 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): Opened 25b6494b58e50dbfdbcd9050bce4992b; next sequenceid=28; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10795346080, jitterRate=0.00539495050907135}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:29:33,745 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(992): Region open journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:29:33,746 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b., pid=52, masterSystemTime=1700357373714 2023-11-19T01:29:33,747 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:33,747 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(158): Opened backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:33,748 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=51 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=OPEN, openSeqNum=28, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:33,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=52, resume processing ppid=51 2023-11-19T01:29:33,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, ppid=51, state=SUCCESS; OpenRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274 in 187 msec 2023-11-19T01:29:33,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-11-19T01:29:33,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN in 344 msec 2023-11-19T01:29:33,754 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357373754"}]},"ts":"1700357373754"} 2023-11-19T01:29:33,756 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-19T01:29:33,758 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-19T01:29:33,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; EnableTableProcedure table=backup:system in 370 msec 2023-11-19T01:29:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-19T01:29:34,001 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-19T01:29:34,001 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:29:34,007 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:29:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:29:34,010 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:29:34,010 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-11-19T01:29:34,012 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:29:34,018 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(246): Backup backup_1700357368330 failed. Backup session finished. Status: FAILURE 2023-11-19T01:29:34,019 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:34,019 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:126) ~[test-classes/:?] ... 41 more 2023-11-19T01:29:34,025 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 2 2023-11-19T01:29:34,045 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39fd1254, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:34,047 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:34,048 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:47656, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:34,052 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:34,053 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52390, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:34,110 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:29:34,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:34,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357374115 (current time:1700357374115). 2023-11-19T01:29:34,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:34,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:29:34,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44edc930, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:34,127 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52402, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=53, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 53 2023-11-19T01:29:34,132 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=53 2023-11-19T01:29:34,133 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:34,136 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot, current working dir path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:34,139 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=53 2023-11-19T01:29:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=53 2023-11-19T01:29:34,548 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:29:34,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b}] 2023-11-19T01:29:34,550 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=54, ppid=53, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:34,701 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:34,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40967 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=54 2023-11-19T01:29:34,702 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:34,703 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.HRegion(2819): Flushing 25b6494b58e50dbfdbcd9050bce4992b 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-11-19T01:29:34,717 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/0627e1f45e764f0a85c1e00380271209 is 256, key is session:backup_1700357368330/session:context/1700357374011/Put/seqid=0 2023-11-19T01:29:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=53 2023-11-19T01:29:35,124 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/0627e1f45e764f0a85c1e00380271209 2023-11-19T01:29:35,131 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/0627e1f45e764f0a85c1e00380271209 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/0627e1f45e764f0a85c1e00380271209 2023-11-19T01:29:35,137 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/0627e1f45e764f0a85c1e00380271209, entries=2, sequenceid=33, filesize=5.2 K 2023-11-19T01:29:35,138 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 25b6494b58e50dbfdbcd9050bce4992b in 435ms, sequenceid=33, compaction requested=true 2023-11-19T01:29:35,138 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-19T01:29:35,139 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.HRegion(2520): Flush status journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:29:35,139 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. for snapshot_backup_system completed. 2023-11-19T01:29:35,139 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:29:35,139 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:29:35,139 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1] hfiles 2023-11-19T01:29:35,139 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 for snapshot=snapshot_backup_system 2023-11-19T01:29:35,139 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/0627e1f45e764f0a85c1e00380271209] hfiles 2023-11-19T01:29:35,139 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44 for snapshot=snapshot_backup_system 2023-11-19T01:29:35,139 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9 for snapshot=snapshot_backup_system 2023-11-19T01:29:35,139 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/0627e1f45e764f0a85c1e00380271209 for snapshot=snapshot_backup_system 2023-11-19T01:29:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=53 2023-11-19T01:29:35,547 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:35,547 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=54}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=54 2023-11-19T01:29:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster(4026): Remote procedure done, pid=54 2023-11-19T01:29:35,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:35,549 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=54, ppid=53, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:35,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=54, resume processing ppid=53 2023-11-19T01:29:35,551 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:29:35,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, ppid=53, state=SUCCESS; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b in 1.0020 sec 2023-11-19T01:29:35,552 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:35,553 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:35,553 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:29:35,553 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:29:35,652 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:29:35,966 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:35,974 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:35,974 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:29:35,976 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=53, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:35,976 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 53 2023-11-19T01:29:35,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=53, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8480 sec 2023-11-19T01:29:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=53 2023-11-19T01:29:36,238 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(115): Backup backup_1700357374081 started at 1700357376237. 2023-11-19T01:29:36,239 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(119): Backup session backup_1700357374081 has been started. 2023-11-19T01:29:36,241 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [ns2:test-17003573419741, test-1700357341974] 2023-11-19T01:29:36,243 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(76): StartCode 1700357337536for backupID backup_1700357374081 2023-11-19T01:29:36,243 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-11-19T01:29:36,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:29:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:29:36,244 INFO [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:29:36,245 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:29:36,246 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:29:36,246 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:29:36,246 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:36,248 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:36,248 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:29:36,248 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:29:36,248 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:29:36,248 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:36,249 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:29:36,249 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:36,249 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:29:36,249 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:29:36,249 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:29:36,249 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:29:36,250 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:29:36,250 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:29:36,251 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:29:36,251 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,40967,1700357333274' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:29:36,254 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:36,254 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:36,254 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:36,254 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:36,254 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:29:36,254 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:36,255 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:36,255 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:36,255 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:36,256 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:36,256 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,40967,1700357333274' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:29:36,256 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7f0371ee[Count = 0] remaining members to acquire global barrier 2023-11-19T01:29:36,256 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:29:36,256 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:36,257 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:36,258 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:36,258 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:29:36,258 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:29:36,258 DEBUG [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:36,258 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:29:36,258 INFO [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357356952 highest: 1700357356991 on jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:36,258 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0:(num 1700357356952) roll requested 2023-11-19T01:29:36,269 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258, exclude list is [], retry=0 2023-11-19T01:29:36,273 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:29:36,276 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 with entries=124, filesize=18.41 KB; new WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 2023-11-19T01:29:36,276 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:29:36,276 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 is not closed yet, will try archiving it next time 2023-11-19T01:29:36,276 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta:.meta(num 1700357356973) roll requested 2023-11-19T01:29:36,277 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357335002 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357335002 2023-11-19T01:29:36,279 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 not finished, retry = 0 2023-11-19T01:29:36,289 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357376277.meta, exclude list is [], retry=0 2023-11-19T01:29:36,292 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:29:36,294 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357356973.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357376277.meta 2023-11-19T01:29:36,294 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:29:36,294 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357356973.meta is not closed yet, will try archiving it next time 2023-11-19T01:29:36,294 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1:(num 1700357356991) roll requested 2023-11-19T01:29:36,305 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294, exclude list is [], retry=0 2023-11-19T01:29:36,309 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:29:36,310 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 2023-11-19T01:29:36,310 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:29:36,310 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 is not closed yet, will try archiving it next time 2023-11-19T01:29:36,318 DEBUG [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-11-19T01:29:36,318 INFO [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357376258 on jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:36,327 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:29:36,327 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,40967,1700357333274' in zk 2023-11-19T01:29:36,329 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:29:36,329 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:29:36,329 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:36,329 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:29:36,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:36,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:36,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:36,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:36,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:36,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:36,332 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:36,332 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:36,332 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:36,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,40967,1700357333274': 2023-11-19T01:29:36,333 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,40967,1700357333274' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:29:36,333 INFO [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:29:36,333 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:29:36,333 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:29:36,333 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:29:36,333 INFO [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:29:36,335 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:29:36,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:36,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:36,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:36,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:36,335 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:29:36,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:29:36,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:36,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:36,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:36,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:36,337 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:36,337 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:36,338 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:36,338 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:36,338 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:36,343 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:29:36,343 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:29:36,344 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:29:36,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:29:36,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:29:36,344 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:29:36,344 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:29:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:29:36,448 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase6.apache.org:40967=1700357337536} newestTimestamps: {jenkins-hbase6.apache.org:40967=1700357356991} 2023-11-19T01:29:36,450 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357336010.meta 2023-11-19T01:29:36,450 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357356973.meta 2023-11-19T01:29:36,450 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357376277.meta 2023-11-19T01:29:36,450 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 2023-11-19T01:29:36,450 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 2023-11-19T01:29:36,450 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 2023-11-19T01:29:36,450 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 2023-11-19T01:29:36,862 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(147): Attempting to copy table info for:test-1700357341974 target: hdfs://localhost:39587/backupUT/backup_1700357374081/default/test-1700357341974 descriptor: 'test-1700357341974', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:36,862 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-19T01:29:36,864 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(152): Starting to write region info for table test-1700357341974 2023-11-19T01:29:37,270 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(159): Finished writing region info for table test-1700357341974 2023-11-19T01:29:37,680 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(147): Attempting to copy table info for:ns2:test-17003573419741 target: hdfs://localhost:39587/backupUT/backup_1700357374081/ns2/test-17003573419741 descriptor: 'ns2:test-17003573419741', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:37,680 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-19T01:29:37,683 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(152): Starting to write region info for table ns2:test-17003573419741 2023-11-19T01:29:37,765 DEBUG [master/jenkins-hbase6:0.Chore.1 {}] procedure2.ProcedureExecutor(1032): Stored pid=55, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE; InitializeStoreFileTrackerProcedure table=test-1700357341974 2023-11-19T01:29:38,088 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(159): Finished writing region info for table ns2:test-17003573419741 2023-11-19T01:29:38,097 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:39587/backupUT/.tmp/backup_1700357374081 from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952;hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 2023-11-19T01:29:38,108 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ef2d49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:38,110 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:38,111 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:47670, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:38,130 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-19T01:29:38,131 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-17003573419741 2023-11-19T01:29:38,136 DEBUG [hconnection-0x66a2eb0-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:38,137 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52408, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:38,139 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-17003573419741: ns2:test-17003573419741; 2023-11-19T01:29:38,140 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1700357341974 2023-11-19T01:29:38,145 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1700357341974: test-1700357341974; 2023-11-19T01:29:38,145 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-11-19T01:29:38,146 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/hbase-staging/partitions_5b6bcf3d-baef-472b-b4e7-432d663dcf81 2023-11-19T01:29:38,179 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/.tabledesc/.tableinfo.0000000002.598 2023-11-19T01:29:38,179 INFO [PEWorker-2 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/.tabledesc/.tableinfo.0000000003.640 2023-11-19T01:29:38,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, state=SUCCESS; InitializeStoreFileTrackerProcedure table=test-1700357341974 in 420 msec 2023-11-19T01:29:38,595 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:38,595 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:38,596 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:38,596 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,588 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop-5548977786917683017.jar 2023-11-19T01:29:39,589 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,589 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,590 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,590 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,590 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,591 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,591 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,592 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,592 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,593 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:39,594 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:29:39,594 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:29:39,594 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:29:39,595 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:29:39,596 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:29:39,596 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:29:39,596 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:29:39,597 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:29:39,597 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:29:39,598 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:29:39,598 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:29:39,600 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,601 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,601 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,602 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,602 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,602 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:39,603 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:39,603 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-17003573419741,default:test-1700357341974 2023-11-19T01:29:39,604 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:39,605 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:40,598 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop-3294686471170382567.jar 2023-11-19T01:29:40,618 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 for WAL files 2023-11-19T01:29:40,620 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 2023-11-19T01:29:40,620 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 for WAL files 2023-11-19T01:29:40,621 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 2023-11-19T01:29:40,700 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.local.dir/1700357380666/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:29:40,700 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.local.dir/1700357380666/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:29:40,786 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 for hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 (-9223372036854775808:9223372036854775807) length:18860 2023-11-19T01:29:40,801 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-19T01:29:40,802 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-19T01:29:40,844 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 for hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 (-9223372036854775808:9223372036854775807) length:1198 2023-11-19T01:29:40,848 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-19T01:29:40,848 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-19T01:29:40,961 DEBUG [pool-226-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338c60bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:40,963 DEBUG [hconnection-0x74f28c6b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:40,966 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:48916, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:40,968 DEBUG [pool-226-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:40,970 DEBUG [pool-226-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-11-19T01:29:40,970 DEBUG [pool-226-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:29:40,975 DEBUG [pool-226-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/backupUT/.tmp/backup_1700357374081/_temporary/0/_temporary/attempt_local156103351_0003_r_000000_0/ns2/test-17003573419741/f/3f0de302d55642c6a502a6ef07457e92 is 38, key is row-t20/f:q1/1700357368147/Put/seqid=0 2023-11-19T01:29:40,977 WARN [Thread-912 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:40967] Specified favored nodes: [jenkins-hbase6.apache.org:40967] 2023-11-19T01:29:41,004 DEBUG [pool-226-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fdd025c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:41,007 DEBUG [hconnection-0x32a76358-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:41,008 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:48922, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:41,010 DEBUG [pool-226-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:41,011 DEBUG [pool-226-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1700357341974-1-0] 2023-11-19T01:29:41,011 DEBUG [pool-226-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:29:41,021 DEBUG [pool-226-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/backupUT/.tmp/backup_1700357374081/_temporary/0/_temporary/attempt_local156103351_0003_r_000001_0/default/test-1700357341974/f/65b8e56b134742618d1854b3ba3f8f8b is 59, key is row-test-1700357341974-1-10/f:q1/1700357368029/Put/seqid=0 2023-11-19T01:29:41,023 WARN [Thread-922 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:40967] Specified favored nodes: [jenkins-hbase6.apache.org:40967] 2023-11-19T01:29:41,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-19T01:29:41,341 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-11-19T01:29:41,737 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:39587/backupUT 2023-11-19T01:29:41,737 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1700357374081 2023-11-19T01:29:41,737 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-11-19T01:29:41,752 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:39587/backupUT/.tmp/backup_1700357374081, hdfs://localhost:39587/backupUT] 2023-11-19T01:29:42,891 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-11-19T01:29:42,893 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1700357374081 2023-11-19T01:29:42,893 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16003 bytes copied." 2023-11-19T01:29:42,893 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local960923983_0004 completed: true true 2023-11-19T01:29:42,899 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=3792132 FILE: Number of bytes written=5076450 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=189942 HDFS: Number of bytes written=2364062 HDFS: Number of read operations=690 HDFS: Number of large read operations=0 HDFS: Number of write operations=246 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1733820416 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16003 Bytes Expected=16003 Files Copied=3 DIR_COPY=7 2023-11-19T01:29:42,899 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:39587/backupUT/.tmp/backup_1700357374081 to hdfs://localhost:39587/backupUT finished. 2023-11-19T01:29:42,900 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_2 in testing java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:29:42,901 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(240): BackupId=backup_1700357374081,startts=1700357376237,failedts=1700357382901,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_2 in testing 2023-11-19T01:29:42,901 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-19T01:29:42,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable backup:system 2023-11-19T01:29:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-19T01:29:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-19T01:29:42,910 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357382910"}]},"ts":"1700357382910"} 2023-11-19T01:29:42,911 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-19T01:29:42,913 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-19T01:29:42,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, UNASSIGN}] 2023-11-19T01:29:42,915 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=57, ppid=56, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, UNASSIGN 2023-11-19T01:29:42,915 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=57 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:42,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE; CloseRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:29:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-19T01:29:43,068 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:43,068 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(119): Close 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:43,068 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1668): Closing 25b6494b58e50dbfdbcd9050bce4992b, disabling compactions & flushes 2023-11-19T01:29:43,068 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1690): Closing region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:43,068 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:43,068 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. after waiting 0 ms 2023-11-19T01:29:43,068 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:43,068 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(2819): Flushing 25b6494b58e50dbfdbcd9050bce4992b 2/2 column families, dataSize=540 B heapSize=1.24 KB 2023-11-19T01:29:43,087 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/meta/277ce0d1b7dc488f8d573b4dc9f97aff is 116, key is rslogts:hdfs://localhost:39587/backupUT\x00jenkins-hbase6.apache.org:40967/meta:rs-log-ts/1700357376326/Put/seqid=0 2023-11-19T01:29:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-19T01:29:43,492 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=112 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/meta/277ce0d1b7dc488f8d573b4dc9f97aff 2023-11-19T01:29:43,501 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/d2bbcdd863fc4593add0403fc78ebedc is 218, key is session:backup_1700357374081/session:context/1700357382892/Put/seqid=0 2023-11-19T01:29:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-19T01:29:43,911 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/d2bbcdd863fc4593add0403fc78ebedc 2023-11-19T01:29:43,919 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/meta/277ce0d1b7dc488f8d573b4dc9f97aff as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/277ce0d1b7dc488f8d573b4dc9f97aff 2023-11-19T01:29:43,925 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/277ce0d1b7dc488f8d573b4dc9f97aff, entries=1, sequenceid=39, filesize=5.3 K 2023-11-19T01:29:43,926 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/d2bbcdd863fc4593add0403fc78ebedc as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d2bbcdd863fc4593add0403fc78ebedc 2023-11-19T01:29:43,932 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d2bbcdd863fc4593add0403fc78ebedc, entries=1, sequenceid=39, filesize=5.2 K 2023-11-19T01:29:43,933 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(3022): Finished flush of dataSize ~540 B/540, heapSize ~1.21 KB/1240, currentSize=0 B/0 for 25b6494b58e50dbfdbcd9050bce4992b in 865ms, sequenceid=39, compaction requested=true 2023-11-19T01:29:43,951 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/recovered.edits/42.seqid, newMaxSeqId=42, maxSeqId=27 2023-11-19T01:29:43,952 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:43,953 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1909): Closed backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:43,953 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1622): Region close journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:29:43,956 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(163): Closed 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:43,956 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=57 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=CLOSED 2023-11-19T01:29:43,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=58, resume processing ppid=57 2023-11-19T01:29:43,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, ppid=57, state=SUCCESS; CloseRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274 in 1.0410 sec 2023-11-19T01:29:43,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=57, resume processing ppid=56 2023-11-19T01:29:43,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, ppid=56, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, UNASSIGN in 1.0450 sec 2023-11-19T01:29:43,961 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357383961"}]},"ts":"1700357383961"} 2023-11-19T01:29:43,962 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-19T01:29:43,965 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-19T01:29:43,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0590 sec 2023-11-19T01:29:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2023-11-19T01:29:44,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357384023 (current time:1700357384023). 2023-11-19T01:29:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1700357384022 VERSION not specified, setting to 2 2023-11-19T01:29:44,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4afffaec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:44,035 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:48930, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=59, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 59 2023-11-19T01:29:44,041 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-19T01:29:44,042 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:44,045 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot, current working dir path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357384022 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:44,047 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-19T01:29:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-19T01:29:44,454 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-19T01:29:44,477 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-19T01:29:44,477 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-19T01:29:44,477 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:44,480 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/277ce0d1b7dc488f8d573b4dc9f97aff, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1] hfiles 2023-11-19T01:29:44,480 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/277ce0d1b7dc488f8d573b4dc9f97aff 2023-11-19T01:29:44,480 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 2023-11-19T01:29:44,480 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:44,482 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/0627e1f45e764f0a85c1e00380271209, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d2bbcdd863fc4593add0403fc78ebedc, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9] hfiles 2023-11-19T01:29:44,482 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/0627e1f45e764f0a85c1e00380271209 2023-11-19T01:29:44,482 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44 2023-11-19T01:29:44,482 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d2bbcdd863fc4593add0403fc78ebedc 2023-11-19T01:29:44,482 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9 2023-11-19T01:29:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-19T01:29:44,888 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-19T01:29:44,889 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:44,890 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:44,890 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1700357384022 2023-11-19T01:29:44,891 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357384022 2023-11-19T01:29:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-19T01:29:45,302 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:45,310 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:45,310 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357384022 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1700357384022 2023-11-19T01:29:45,312 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:45,312 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 59 2023-11-19T01:29:45,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357384022 table=backup:system type=FLUSH ttl=0 } in 1.2750 sec 2023-11-19T01:29:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=59 2023-11-19T01:29:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=60, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357374115 type: FLUSH version: 2 ttl: 0 ) 2023-11-19T01:29:46,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-19T01:29:46,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-19T01:29:46,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-19T01:29:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-19T01:29:46,572 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-11-19T01:29:46,572 INFO [PEWorker-1 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-11-19T01:29:46,573 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:46,576 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357374115 type: FLUSH version: 2 ttl: 0 2023-11-19T01:29:46,576 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system 2023-11-19T01:29:46,579 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-19T01:29:46,579 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:46,585 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/277ce0d1b7dc488f8d573b4dc9f97aff to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/archive/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/277ce0d1b7dc488f8d573b4dc9f97aff 2023-11-19T01:29:46,585 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:46,589 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d2bbcdd863fc4593add0403fc78ebedc to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/archive/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d2bbcdd863fc4593add0403fc78ebedc 2023-11-19T01:29:46,589 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:46,589 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357374115 type: FLUSH version: 2 ttl: 0 2023-11-19T01:29:46,590 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357386590"}]},"ts":"9223372036854775807"} 2023-11-19T01:29:46,592 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:29:46,592 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:29:46,592 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1700357386591"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357386591"}]},"ts":"1700357386591"} 2023-11-19T01:29:46,594 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:29:46,594 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-19T01:29:46,594 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:29:46,594 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-19T01:29:46,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357374115 type: FLUSH version: 2 ttl: 0 ) in 437 msec 2023-11-19T01:29:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-19T01:29:46,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "hbase-failsafe-snapshot_backup_system-1700357384022" 2023-11-19T01:29:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1700357384022 2023-11-19T01:29:46,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$12(2662): Client=jenkins//172.31.2.20 enable backup:system 2023-11-19T01:29:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=61, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-19T01:29:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=61 2023-11-19T01:29:46,772 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-19T01:29:46,772 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357386772"}]},"ts":"1700357386772"} 2023-11-19T01:29:46,774 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-19T01:29:46,776 DEBUG [PEWorker-5 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-19T01:29:46,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN}] 2023-11-19T01:29:46,778 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN 2023-11-19T01:29:46,779 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-19T01:29:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=61 2023-11-19T01:29:46,930 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=62 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:46,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; OpenRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:29:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=61 2023-11-19T01:29:47,083 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:47,086 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(130): Open backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:47,087 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7254): Opening region: {ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:29:47,087 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:29:47,087 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:47,087 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(888): Instantiated backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:47,087 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7296): checking encryption for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:47,087 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7299): checking classloading for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:47,088 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:47,089 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25b6494b58e50dbfdbcd9050bce4992b columnFamilyName meta 2023-11-19T01:29:47,089 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:47,095 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 2023-11-19T01:29:47,095 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(324): Store=25b6494b58e50dbfdbcd9050bce4992b/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:47,096 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:47,096 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25b6494b58e50dbfdbcd9050bce4992b columnFamilyName session 2023-11-19T01:29:47,096 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:47,103 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/0627e1f45e764f0a85c1e00380271209 2023-11-19T01:29:47,107 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44 2023-11-19T01:29:47,111 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9 2023-11-19T01:29:47,111 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(324): Store=25b6494b58e50dbfdbcd9050bce4992b/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:47,112 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:47,113 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:47,115 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:29:47,116 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1076): writing seq id for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:47,117 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): Opened 25b6494b58e50dbfdbcd9050bce4992b; next sequenceid=43; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10205320960, jitterRate=-0.049555420875549316}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:29:47,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(992): Region open journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:29:47,119 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b., pid=63, masterSystemTime=1700357387083 2023-11-19T01:29:47,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.CompactSplit(403): Add compact mark for store 25b6494b58e50dbfdbcd9050bce4992b:session, priority=-2147483648, current under compaction store size is 1 2023-11-19T01:29:47,122 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-11-19T01:29:47,123 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-11-19T01:29:47,124 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:47,124 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(158): Opened backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:47,125 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=62 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=OPEN, openSeqNum=43, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:47,126 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 16169 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-11-19T01:29:47,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-11-19T01:29:47,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; OpenRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274 in 195 msec 2023-11-19T01:29:47,128 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.HStore(1534): 25b6494b58e50dbfdbcd9050bce4992b/session is initiating minor compaction (all files) 2023-11-19T01:29:47,128 INFO [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 25b6494b58e50dbfdbcd9050bce4992b/session in backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:47,128 INFO [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/0627e1f45e764f0a85c1e00380271209] into tmpdir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp, totalSize=15.8 K 2023-11-19T01:29:47,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=62, resume processing ppid=61 2023-11-19T01:29:47,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, ppid=61, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN in 352 msec 2023-11-19T01:29:47,130 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1b3b4537222a44d3a2ca425154b42b44, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1700357354741 2023-11-19T01:29:47,130 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357387130"}]},"ts":"1700357387130"} 2023-11-19T01:29:47,130 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] compactions.Compactor(223): Compacting d9d812f8f01b44aa909e0990967aeae9, keycount=3, bloomtype=ROW, size=5.7 K, encoding=NONE, compression=NONE, seqNum=20, earliestPutTs=1700357367978 2023-11-19T01:29:47,131 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0627e1f45e764f0a85c1e00380271209, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1700357374011 2023-11-19T01:29:47,131 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-19T01:29:47,134 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-19T01:29:47,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, state=SUCCESS; EnableTableProcedure table=backup:system in 365 msec 2023-11-19T01:29:47,156 INFO [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25b6494b58e50dbfdbcd9050bce4992b#session#compaction#10 average throughput is 0.95 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-11-19T01:29:47,157 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/570e5efe080c4c40a868ff08386fd609 is 459, key is session:backup_1700357350318/session:context/1700357367978/Put/seqid=0 2023-11-19T01:29:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=61 2023-11-19T01:29:47,374 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-19T01:29:47,374 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:29:47,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:29:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:29:47,382 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:29:47,382 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-11-19T01:29:47,383 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:39587/backupUT/backup_1700357374081/default/test-1700357341974 done. 2023-11-19T01:29:47,384 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(208): hdfs://localhost:39587/backupUT/backup_1700357374081/default is empty, remove it. 2023-11-19T01:29:47,384 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:39587/backupUT/backup_1700357374081/ns2/test-17003573419741 done. 2023-11-19T01:29:47,385 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(208): hdfs://localhost:39587/backupUT/backup_1700357374081/ns2 is empty, remove it. 2023-11-19T01:29:47,386 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:29:47,393 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(246): Backup backup_1700357374081 failed. Backup session finished. Status: FAILURE 2023-11-19T01:29:47,393 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:47,394 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:137) ~[test-classes/:?] ... 41 more 2023-11-19T01:29:47,398 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 3 2023-11-19T01:29:47,419 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6898046f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:47,420 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:47,422 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:33506, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:47,426 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:47,428 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:48940, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:47,474 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:29:47,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357387477 (current time:1700357387477). 2023-11-19T01:29:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:29:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f35caf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:47,486 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:48952, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-11-19T01:29:47,493 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:29:47,494 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:47,497 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot, current working dir path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:47,500 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:47,568 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/570e5efe080c4c40a868ff08386fd609 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609 2023-11-19T01:29:47,586 INFO [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 25b6494b58e50dbfdbcd9050bce4992b/session of 25b6494b58e50dbfdbcd9050bce4992b into 570e5efe080c4c40a868ff08386fd609(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-11-19T01:29:47,586 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:29:47,586 INFO [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b., storeName=25b6494b58e50dbfdbcd9050bce4992b/session, priority=13, startTime=1700357387121; duration=0sec 2023-11-19T01:29:47,587 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-11-19T01:29:47,587 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25b6494b58e50dbfdbcd9050bce4992b:session 2023-11-19T01:29:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:29:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:29:47,909 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:29:47,909 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b}] 2023-11-19T01:29:47,911 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:48,031 INFO [master/jenkins-hbase6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-11-19T01:29:48,031 INFO [master/jenkins-hbase6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-11-19T01:29:48,062 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:48,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40967 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-11-19T01:29:48,063 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:48,063 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing 25b6494b58e50dbfdbcd9050bce4992b 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-11-19T01:29:48,067 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/f137c14c96c8462d80e28a3ba7fb085c is 256, key is session:backup_1700357374081/session:context/1700357387386/Put/seqid=0 2023-11-19T01:29:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:29:48,471 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/f137c14c96c8462d80e28a3ba7fb085c 2023-11-19T01:29:48,478 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/f137c14c96c8462d80e28a3ba7fb085c as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c 2023-11-19T01:29:48,483 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c, entries=2, sequenceid=49, filesize=5.2 K 2023-11-19T01:29:48,484 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 25b6494b58e50dbfdbcd9050bce4992b in 421ms, sequenceid=49, compaction requested=false 2023-11-19T01:29:48,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:29:48,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. for snapshot_backup_system completed. 2023-11-19T01:29:48,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:29:48,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:29:48,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1] hfiles 2023-11-19T01:29:48,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 for snapshot=snapshot_backup_system 2023-11-19T01:29:48,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c] hfiles 2023-11-19T01:29:48,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609 for snapshot=snapshot_backup_system 2023-11-19T01:29:48,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c for snapshot=snapshot_backup_system 2023-11-19T01:29:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:29:48,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:48,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-11-19T01:29:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster(4026): Remote procedure done, pid=65 2023-11-19T01:29:48,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:48,895 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:48,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-11-19T01:29:48,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b in 987 msec 2023-11-19T01:29:48,897 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:29:48,898 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:48,898 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:48,898 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:29:48,899 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:29:49,311 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:49,318 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:49,318 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:29:49,320 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:49,320 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-11-19T01:29:49,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8320 sec 2023-11-19T01:29:49,506 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:29:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:29:49,599 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(115): Backup backup_1700357387449 started at 1700357389599. 2023-11-19T01:29:49,600 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(119): Backup session backup_1700357387449 has been started. 2023-11-19T01:29:49,602 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [ns2:test-17003573419741, test-1700357341974] 2023-11-19T01:29:49,605 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(76): StartCode 1700357337536for backupID backup_1700357387449 2023-11-19T01:29:49,605 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-11-19T01:29:49,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:29:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:29:49,606 INFO [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:29:49,606 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:29:49,607 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:29:49,607 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:29:49,608 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:49,610 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:49,610 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:29:49,610 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:29:49,611 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:29:49,611 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:49,611 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:29:49,611 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:49,611 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:29:49,612 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:29:49,612 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:29:49,612 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:29:49,612 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:29:49,613 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:29:49,613 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:29:49,613 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,40967,1700357333274' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:29:49,616 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:49,616 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:49,616 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:49,616 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:49,616 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:29:49,616 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:49,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:49,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:49,617 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:49,618 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:49,618 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,40967,1700357333274' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:29:49,618 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@159ab902[Count = 0] remaining members to acquire global barrier 2023-11-19T01:29:49,618 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:29:49,618 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:49,621 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:49,621 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:49,621 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:29:49,621 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:29:49,622 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:29:49,622 DEBUG [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:49,622 INFO [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357376258 highest: 1700357376294 on jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:49,622 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0:(num 1700357376258) roll requested 2023-11-19T01:29:49,634 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357389622, exclude list is [], retry=0 2023-11-19T01:29:49,637 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:29:49,639 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 with entries=13, filesize=3.56 KB; new WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357389622 2023-11-19T01:29:49,639 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:29:49,639 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 is not closed yet, will try archiving it next time 2023-11-19T01:29:49,639 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta:.meta(num 1700357376277) roll requested 2023-11-19T01:29:49,641 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 not finished, retry = 0 2023-11-19T01:29:49,650 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357389639.meta, exclude list is [], retry=0 2023-11-19T01:29:49,653 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:29:49,655 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357376277.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357389639.meta 2023-11-19T01:29:49,655 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:29:49,655 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357376277.meta is not closed yet, will try archiving it next time 2023-11-19T01:29:49,655 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1:(num 1700357376294) roll requested 2023-11-19T01:29:49,666 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357389655, exclude list is [], retry=0 2023-11-19T01:29:49,669 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:29:49,671 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357389655 2023-11-19T01:29:49,671 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:29:49,671 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 is not closed yet, will try archiving it next time 2023-11-19T01:29:49,673 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 not finished, retry = 0 2023-11-19T01:29:49,682 DEBUG [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-11-19T01:29:49,682 INFO [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357389622 on jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:49,690 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:29:49,690 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,40967,1700357333274' in zk 2023-11-19T01:29:49,692 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:29:49,692 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:29:49,692 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:29:49,693 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:49,693 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:49,693 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:49,694 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:49,694 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:49,694 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:49,695 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:49,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:49,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:49,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:49,697 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,40967,1700357333274': 2023-11-19T01:29:49,697 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,40967,1700357333274' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:29:49,697 INFO [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:29:49,697 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:29:49,697 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:29:49,697 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:29:49,697 INFO [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:29:49,699 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:29:49,699 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:49,700 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:49,700 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:49,700 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:49,700 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:29:49,700 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:29:49,700 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:49,700 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:49,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:49,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:49,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:49,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:49,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:49,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:49,703 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:49,707 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:29:49,707 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:29:49,707 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:29:49,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:29:49,707 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:29:49,708 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:29:49,708 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:29:49,774 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 2023-11-19T01:29:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:29:49,810 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase6.apache.org:40967=1700357337536} newestTimestamps: {jenkins-hbase6.apache.org:40967=1700357376294} 2023-11-19T01:29:49,813 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357336010.meta 2023-11-19T01:29:49,813 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357356973.meta 2023-11-19T01:29:49,813 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357376277.meta 2023-11-19T01:29:49,813 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357389639.meta 2023-11-19T01:29:49,813 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 2023-11-19T01:29:49,813 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 2023-11-19T01:29:49,813 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357389622 2023-11-19T01:29:49,813 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 2023-11-19T01:29:49,813 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357389655 2023-11-19T01:29:50,224 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(147): Attempting to copy table info for:test-1700357341974 target: hdfs://localhost:39587/backupUT/backup_1700357387449/default/test-1700357341974 descriptor: 'test-1700357341974', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:50,224 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-19T01:29:50,226 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(152): Starting to write region info for table test-1700357341974 2023-11-19T01:29:50,632 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(159): Finished writing region info for table test-1700357341974 2023-11-19T01:29:51,044 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(147): Attempting to copy table info for:ns2:test-17003573419741 target: hdfs://localhost:39587/backupUT/backup_1700357387449/ns2/test-17003573419741 descriptor: 'ns2:test-17003573419741', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:51,044 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-19T01:29:51,046 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(152): Starting to write region info for table ns2:test-17003573419741 2023-11-19T01:29:51,452 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(159): Finished writing region info for table ns2:test-17003573419741 2023-11-19T01:29:51,459 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:39587/backupUT/.tmp/backup_1700357387449 from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952;hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258;hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991;hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 2023-11-19T01:29:51,469 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@212e19af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:51,470 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:51,472 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49194, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:51,474 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-19T01:29:51,474 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-17003573419741 2023-11-19T01:29:51,477 DEBUG [hconnection-0x4128f818-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:51,479 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49132, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:51,481 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-17003573419741: ns2:test-17003573419741; 2023-11-19T01:29:51,481 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1700357341974 2023-11-19T01:29:51,484 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1700357341974: test-1700357341974; 2023-11-19T01:29:51,484 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-11-19T01:29:51,485 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/hbase-staging/partitions_d6b57298-592b-4b01-90bf-7428b68d7c5f 2023-11-19T01:29:51,493 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:51,494 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:51,495 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:51,495 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,595 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop-3909899223311223076.jar 2023-11-19T01:29:52,595 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,596 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,597 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,597 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,598 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,599 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,599 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,600 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,600 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,601 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:52,602 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:29:52,603 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:29:52,603 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:29:52,604 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:29:52,604 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:29:52,604 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:29:52,605 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:29:52,605 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:29:52,606 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:29:52,606 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:29:52,607 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:29:52,607 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,608 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,609 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,609 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,609 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,610 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:52,611 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:52,611 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-17003573419741,default:test-1700357341974 2023-11-19T01:29:52,611 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:52,612 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:53,646 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop-1971631237130132461.jar 2023-11-19T01:29:53,669 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 for WAL files 2023-11-19T01:29:53,671 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 2023-11-19T01:29:53,671 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 for WAL files 2023-11-19T01:29:53,672 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 2023-11-19T01:29:53,672 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 for WAL files 2023-11-19T01:29:53,673 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 2023-11-19T01:29:53,673 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 for WAL files 2023-11-19T01:29:53,674 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 2023-11-19T01:29:53,757 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.local.dir/1700357393714/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:29:53,757 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.local.dir/1700357393714/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:29:53,825 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 for hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 (-9223372036854775808:9223372036854775807) length:18860 2023-11-19T01:29:53,833 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-19T01:29:53,834 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-19T01:29:53,859 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 for hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 (-9223372036854775808:9223372036854775807) length:3657 2023-11-19T01:29:53,862 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-19T01:29:53,862 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-19T01:29:53,886 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 for hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 (-9223372036854775808:9223372036854775807) length:1198 2023-11-19T01:29:53,889 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-19T01:29:53,889 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-19T01:29:53,937 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 for hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 (-9223372036854775808:9223372036854775807) length:93 2023-11-19T01:29:53,940 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-19T01:29:53,940 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-19T01:29:53,968 DEBUG [pool-249-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64a08d9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:53,972 DEBUG [hconnection-0xdef7396-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:53,973 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49134, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:53,974 DEBUG [pool-249-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:53,976 DEBUG [pool-249-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-11-19T01:29:53,976 DEBUG [pool-249-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:29:53,988 DEBUG [pool-249-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/backupUT/.tmp/backup_1700357387449/_temporary/0/_temporary/attempt_local179593962_0005_r_000000_0/ns2/test-17003573419741/f/8d8a0a900199429399f3afd936812cb5 is 38, key is row-t20/f:q1/1700357368147/Put/seqid=0 2023-11-19T01:29:53,990 WARN [Thread-1078 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:40967] Specified favored nodes: [jenkins-hbase6.apache.org:40967] 2023-11-19T01:29:54,447 DEBUG [pool-249-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2892ee29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:54,449 DEBUG [hconnection-0x2030a37-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:54,451 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49138, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:54,453 DEBUG [pool-249-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:54,457 DEBUG [pool-249-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1700357341974-1-0] 2023-11-19T01:29:54,457 DEBUG [pool-249-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:29:54,468 DEBUG [pool-249-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/backupUT/.tmp/backup_1700357387449/_temporary/0/_temporary/attempt_local179593962_0005_r_000001_0/default/test-1700357341974/f/a2c1b3b4ba594ddabf2e8ce5e9839320 is 59, key is row-test-1700357341974-1-10/f:q1/1700357368029/Put/seqid=0 2023-11-19T01:29:54,470 WARN [Thread-1088 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:40967] Specified favored nodes: [jenkins-hbase6.apache.org:40967] 2023-11-19T01:29:55,812 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:39587/backupUT 2023-11-19T01:29:55,812 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1700357387449 2023-11-19T01:29:55,812 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-11-19T01:29:55,819 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:39587/backupUT/.tmp/backup_1700357387449, hdfs://localhost:39587/backupUT] 2023-11-19T01:29:56,914 DEBUG [master/jenkins-hbase6:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 36f383735ae1fecc54a98975fe8d38af changed from -1.0 to 0.0, refreshing cache 2023-11-19T01:29:56,924 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-11-19T01:29:56,926 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1700357387449 2023-11-19T01:29:56,926 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16003 bytes copied." 2023-11-19T01:29:56,926 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1500842329_0006 completed: true true 2023-11-19T01:29:56,928 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=4449500 FILE: Number of bytes written=7501600 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=299720 HDFS: Number of bytes written=2422518 HDFS: Number of read operations=1016 HDFS: Number of large read operations=0 HDFS: Number of write operations=326 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1729101824 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16003 Bytes Expected=16003 Files Copied=3 DIR_COPY=7 2023-11-19T01:29:56,928 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:39587/backupUT/.tmp/backup_1700357387449 to hdfs://localhost:39587/backupUT finished. 2023-11-19T01:29:56,933 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_3 in testing java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:150) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:29:56,934 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(240): BackupId=backup_1700357387449,startts=1700357389599,failedts=1700357396934,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_3 in testing 2023-11-19T01:29:56,934 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-19T01:29:56,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable backup:system 2023-11-19T01:29:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-19T01:29:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:29:56,942 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357396942"}]},"ts":"1700357396942"} 2023-11-19T01:29:56,943 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-19T01:29:56,946 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-19T01:29:56,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, UNASSIGN}] 2023-11-19T01:29:56,951 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, UNASSIGN 2023-11-19T01:29:56,951 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:56,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; CloseRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:29:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:29:57,104 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:29:57,105 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] handler.UnassignRegionHandler(119): Close 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:57,105 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1668): Closing 25b6494b58e50dbfdbcd9050bce4992b, disabling compactions & flushes 2023-11-19T01:29:57,105 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1690): Closing region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:57,105 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:57,105 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. after waiting 0 ms 2023-11-19T01:29:57,105 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:57,105 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(2819): Flushing 25b6494b58e50dbfdbcd9050bce4992b 2/2 column families, dataSize=872 B heapSize=1.71 KB 2023-11-19T01:29:57,123 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/meta/c228097b990b49e7bcd58c0a28f2b1ce is 171, key is trslm:hdfs://localhost:39587/backupUT\x00ns2:test-17003573419741/meta:log-roll-map/1700357396932/Put/seqid=0 2023-11-19T01:29:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:29:57,527 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=444 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/meta/c228097b990b49e7bcd58c0a28f2b1ce 2023-11-19T01:29:57,539 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/78231561ea8241a9b1c316d2babf4d54 is 218, key is session:backup_1700357387449/session:context/1700357396925/Put/seqid=0 2023-11-19T01:29:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:29:57,944 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/78231561ea8241a9b1c316d2babf4d54 2023-11-19T01:29:57,950 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/meta/c228097b990b49e7bcd58c0a28f2b1ce as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/c228097b990b49e7bcd58c0a28f2b1ce 2023-11-19T01:29:57,956 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/c228097b990b49e7bcd58c0a28f2b1ce, entries=3, sequenceid=56, filesize=5.6 K 2023-11-19T01:29:57,957 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/78231561ea8241a9b1c316d2babf4d54 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/78231561ea8241a9b1c316d2babf4d54 2023-11-19T01:29:57,963 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/78231561ea8241a9b1c316d2babf4d54, entries=1, sequenceid=56, filesize=5.2 K 2023-11-19T01:29:57,964 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(3022): Finished flush of dataSize ~872 B/872, heapSize ~1.68 KB/1720, currentSize=0 B/0 for 25b6494b58e50dbfdbcd9050bce4992b in 859ms, sequenceid=56, compaction requested=true 2023-11-19T01:29:57,973 DEBUG [StoreCloser-backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/0627e1f45e764f0a85c1e00380271209] to archive 2023-11-19T01:29:57,974 DEBUG [StoreCloser-backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-11-19T01:29:57,978 DEBUG [StoreCloser-backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/archive/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/1b3b4537222a44d3a2ca425154b42b44 2023-11-19T01:29:57,979 DEBUG [StoreCloser-backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/archive/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/d9d812f8f01b44aa909e0990967aeae9 2023-11-19T01:29:57,980 DEBUG [StoreCloser-backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/0627e1f45e764f0a85c1e00380271209 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/archive/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/0627e1f45e764f0a85c1e00380271209 2023-11-19T01:29:58,016 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=42 2023-11-19T01:29:58,017 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:58,018 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1909): Closed backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:29:58,018 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1622): Region close journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:29:58,019 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] handler.UnassignRegionHandler(163): Closed 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:29:58,020 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=CLOSED 2023-11-19T01:29:58,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=68, resume processing ppid=67 2023-11-19T01:29:58,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, ppid=67, state=SUCCESS; CloseRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274 in 1.0690 sec 2023-11-19T01:29:58,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-11-19T01:29:58,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, UNASSIGN in 1.0750 sec 2023-11-19T01:29:58,024 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357398024"}]},"ts":"1700357398024"} 2023-11-19T01:29:58,025 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-19T01:29:58,028 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-19T01:29:58,029 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0900 sec 2023-11-19T01:29:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:29:58,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:58,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357398058 (current time:1700357398058). 2023-11-19T01:29:58,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:58,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1700357398058 VERSION not specified, setting to 2 2023-11-19T01:29:58,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@92452a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:58,067 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49140, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:58,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=69, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 69 2023-11-19T01:29:58,071 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2023-11-19T01:29:58,072 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:58,074 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot, current working dir path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357398058 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:58,076 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:58,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2023-11-19T01:29:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2023-11-19T01:29:58,483 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-19T01:29:58,484 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-19T01:29:58,484 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-19T01:29:58,484 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:58,486 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/c228097b990b49e7bcd58c0a28f2b1ce] hfiles 2023-11-19T01:29:58,486 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 2023-11-19T01:29:58,486 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/c228097b990b49e7bcd58c0a28f2b1ce 2023-11-19T01:29:58,486 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:58,488 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/78231561ea8241a9b1c316d2babf4d54, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c] hfiles 2023-11-19T01:29:58,488 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609 2023-11-19T01:29:58,488 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/78231561ea8241a9b1c316d2babf4d54 2023-11-19T01:29:58,488 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c 2023-11-19T01:29:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2023-11-19T01:29:58,894 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-19T01:29:58,895 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:58,896 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:58,896 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1700357398058 2023-11-19T01:29:58,896 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357398058 2023-11-19T01:29:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2023-11-19T01:29:59,310 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:59,318 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:59,318 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357398058 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1700357398058 2023-11-19T01:29:59,319 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:59,319 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 69 2023-11-19T01:29:59,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357398058 table=backup:system type=FLUSH ttl=0 } in 1.2510 sec 2023-11-19T01:30:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2023-11-19T01:30:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=70, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357387477 type: FLUSH version: 2 ttl: 0 ) 2023-11-19T01:30:00,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-19T01:30:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-19T01:30:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-19T01:30:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-19T01:30:00,600 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-11-19T01:30:00,600 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-11-19T01:30:00,602 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:30:00,605 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357387477 type: FLUSH version: 2 ttl: 0 2023-11-19T01:30:00,605 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system 2023-11-19T01:30:00,609 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-19T01:30:00,609 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:00,617 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/c228097b990b49e7bcd58c0a28f2b1ce to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/archive/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/c228097b990b49e7bcd58c0a28f2b1ce 2023-11-19T01:30:00,617 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:00,621 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/78231561ea8241a9b1c316d2babf4d54 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/archive/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/78231561ea8241a9b1c316d2babf4d54 2023-11-19T01:30:00,621 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:00,621 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357387477 type: FLUSH version: 2 ttl: 0 2023-11-19T01:30:00,623 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357400623"}]},"ts":"9223372036854775807"} 2023-11-19T01:30:00,629 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:30:00,629 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:30:00,630 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1700357400624"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357400624"}]},"ts":"1700357400624"} 2023-11-19T01:30:00,631 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:30:00,632 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-19T01:30:00,632 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:30:00,632 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-19T01:30:00,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357387477 type: FLUSH version: 2 ttl: 0 ) in 447 msec 2023-11-19T01:30:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-19T01:30:00,793 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "hbase-failsafe-snapshot_backup_system-1700357398058" 2023-11-19T01:30:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1700357398058 2023-11-19T01:30:00,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$12(2662): Client=jenkins//172.31.2.20 enable backup:system 2023-11-19T01:30:00,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=71, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-19T01:30:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-19T01:30:00,802 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-19T01:30:00,802 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357400802"}]},"ts":"1700357400802"} 2023-11-19T01:30:00,803 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-19T01:30:00,806 DEBUG [PEWorker-1 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-19T01:30:00,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN}] 2023-11-19T01:30:00,808 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN 2023-11-19T01:30:00,809 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-19T01:30:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-19T01:30:00,961 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=72 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:00,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE; OpenRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:30:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-19T01:30:01,114 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:01,118 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(130): Open backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:01,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7254): Opening region: {ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:01,118 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:01,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:01,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(888): Instantiated backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:01,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7296): checking encryption for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:01,118 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7299): checking classloading for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:01,120 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:01,120 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25b6494b58e50dbfdbcd9050bce4992b columnFamilyName meta 2023-11-19T01:30:01,120 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:01,126 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 2023-11-19T01:30:01,126 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(324): Store=25b6494b58e50dbfdbcd9050bce4992b/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:01,126 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:01,127 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25b6494b58e50dbfdbcd9050bce4992b columnFamilyName session 2023-11-19T01:30:01,127 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:01,132 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609 2023-11-19T01:30:01,136 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c 2023-11-19T01:30:01,136 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(324): Store=25b6494b58e50dbfdbcd9050bce4992b/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:01,137 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:01,138 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:01,139 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:30:01,141 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1076): writing seq id for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:01,142 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): Opened 25b6494b58e50dbfdbcd9050bce4992b; next sequenceid=60; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11734624320, jitterRate=0.09287205338478088}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:30:01,142 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(992): Region open journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:30:01,143 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b., pid=73, masterSystemTime=1700357401114 2023-11-19T01:30:01,144 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:01,144 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(158): Opened backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:01,145 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=72 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=OPEN, openSeqNum=60, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:01,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=73, resume processing ppid=72 2023-11-19T01:30:01,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, ppid=72, state=SUCCESS; OpenRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274 in 184 msec 2023-11-19T01:30:01,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=72, resume processing ppid=71 2023-11-19T01:30:01,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, ppid=71, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN in 342 msec 2023-11-19T01:30:01,150 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357401149"}]},"ts":"1700357401149"} 2023-11-19T01:30:01,151 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-19T01:30:01,153 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-19T01:30:01,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, state=SUCCESS; EnableTableProcedure table=backup:system in 355 msec 2023-11-19T01:30:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2023-11-19T01:30:01,405 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-19T01:30:01,405 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:30:01,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:30:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:30:01,413 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:30:01,413 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-11-19T01:30:01,414 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:39587/backupUT/backup_1700357387449/default/test-1700357341974 done. 2023-11-19T01:30:01,415 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(208): hdfs://localhost:39587/backupUT/backup_1700357387449/default is empty, remove it. 2023-11-19T01:30:01,415 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:39587/backupUT/backup_1700357387449/ns2/test-17003573419741 done. 2023-11-19T01:30:01,416 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(208): hdfs://localhost:39587/backupUT/backup_1700357387449/ns2 is empty, remove it. 2023-11-19T01:30:01,418 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:30:01,425 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(246): Backup backup_1700357387449 failed. Backup session finished. Status: FAILURE 2023-11-19T01:30:01,426 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:01,426 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:150) ~[test-classes/:?] ... 41 more 2023-11-19T01:30:01,433 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 4 2023-11-19T01:30:01,454 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9b583c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:01,455 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:30:01,456 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52796, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:30:01,459 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:01,460 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39462, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:01,515 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:30:01,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:30:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357401519 (current time:1700357401519). 2023-11-19T01:30:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:30:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:30:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@296776b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:01,530 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39470, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:30:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=74, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:30:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 74 2023-11-19T01:30:01,535 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:30:01,536 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:30:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=74 2023-11-19T01:30:01,539 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot, current working dir path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:30:01,542 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:30:01,562 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:30:01,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b}] 2023-11-19T01:30:01,564 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=74 2023-11-19T01:30:01,715 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:01,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40967 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=75 2023-11-19T01:30:01,716 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:01,717 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(2819): Flushing 25b6494b58e50dbfdbcd9050bce4992b 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-11-19T01:30:01,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/f5e1c45129834a5a845838abfd3e2dad is 256, key is session:backup_1700357387449/session:context/1700357401417/Put/seqid=0 2023-11-19T01:30:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=74 2023-11-19T01:30:02,139 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/f5e1c45129834a5a845838abfd3e2dad 2023-11-19T01:30:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=74 2023-11-19T01:30:02,145 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/f5e1c45129834a5a845838abfd3e2dad as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f5e1c45129834a5a845838abfd3e2dad 2023-11-19T01:30:02,151 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f5e1c45129834a5a845838abfd3e2dad, entries=2, sequenceid=65, filesize=5.2 K 2023-11-19T01:30:02,152 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 25b6494b58e50dbfdbcd9050bce4992b in 435ms, sequenceid=65, compaction requested=true 2023-11-19T01:30:02,153 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-19T01:30:02,154 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(2520): Flush status journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:30:02,154 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. for snapshot_backup_system completed. 2023-11-19T01:30:02,154 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:30:02,155 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:30:02,155 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1] hfiles 2023-11-19T01:30:02,155 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 for snapshot=snapshot_backup_system 2023-11-19T01:30:02,155 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f5e1c45129834a5a845838abfd3e2dad] hfiles 2023-11-19T01:30:02,155 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609 for snapshot=snapshot_backup_system 2023-11-19T01:30:02,155 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c for snapshot=snapshot_backup_system 2023-11-19T01:30:02,155 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f5e1c45129834a5a845838abfd3e2dad for snapshot=snapshot_backup_system 2023-11-19T01:30:02,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:02,162 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=75 2023-11-19T01:30:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster(4026): Remote procedure done, pid=75 2023-11-19T01:30:02,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:02,164 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:02,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=75, resume processing ppid=74 2023-11-19T01:30:02,166 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:30:02,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, ppid=74, state=SUCCESS; SnapshotRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b in 601 msec 2023-11-19T01:30:02,166 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:30:02,167 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:30:02,167 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:30:02,168 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:30:02,578 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:30:02,586 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:30:02,587 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:30:02,588 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:30:02,588 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 74 2023-11-19T01:30:02,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.0570 sec 2023-11-19T01:30:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=74 2023-11-19T01:30:02,642 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(115): Backup backup_1700357401487 started at 1700357402642. 2023-11-19T01:30:02,644 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(119): Backup session backup_1700357401487 has been started. 2023-11-19T01:30:02,645 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [ns2:test-17003573419741, test-1700357341974] 2023-11-19T01:30:02,648 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(76): StartCode 1700357337536for backupID backup_1700357401487 2023-11-19T01:30:02,648 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-11-19T01:30:02,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:30:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:30:02,649 INFO [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:30:02,649 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:30:02,650 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:30:02,650 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:30:02,651 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:30:02,653 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:02,653 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:30:02,653 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:30:02,653 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:30:02,654 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:30:02,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:30:02,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:30:02,654 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:30:02,655 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:30:02,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:30:02,655 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:30:02,655 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:30:02,656 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:30:02,656 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:30:02,656 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,40967,1700357333274' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:30:02,659 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:02,659 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:02,659 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:30:02,659 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:30:02,660 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:30:02,660 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:30:02,660 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:30:02,660 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:02,661 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:02,661 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:30:02,661 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,40967,1700357333274' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:30:02,661 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@591e96ea[Count = 0] remaining members to acquire global barrier 2023-11-19T01:30:02,661 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:30:02,662 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:02,663 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:02,663 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:30:02,663 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:30:02,663 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:30:02,663 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:30:02,664 DEBUG [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:02,664 INFO [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357389622 highest: 1700357389655 on jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:02,664 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0:(num 1700357389622) roll requested 2023-11-19T01:30:02,675 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357402664, exclude list is [], retry=0 2023-11-19T01:30:02,681 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:30:02,683 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357389622 with entries=13, filesize=3.55 KB; new WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357402664 2023-11-19T01:30:02,683 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:30:02,683 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357389622 is not closed yet, will try archiving it next time 2023-11-19T01:30:02,683 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta:.meta(num 1700357389639) roll requested 2023-11-19T01:30:02,683 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 2023-11-19T01:30:02,700 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357402683.meta, exclude list is [], retry=0 2023-11-19T01:30:02,703 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:30:02,705 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357389639.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357402683.meta 2023-11-19T01:30:02,705 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:30:02,706 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357389639.meta is not closed yet, will try archiving it next time 2023-11-19T01:30:02,706 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1:(num 1700357389655) roll requested 2023-11-19T01:30:02,719 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357402706, exclude list is [], retry=0 2023-11-19T01:30:02,723 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK] 2023-11-19T01:30:02,725 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357389655 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357402706 2023-11-19T01:30:02,725 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:41437,DS-411aec13-6b5e-41f2-9713-9adb5ce871b9,DISK]] 2023-11-19T01:30:02,725 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357389655 is not closed yet, will try archiving it next time 2023-11-19T01:30:02,727 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357389655 not finished, retry = 0 2023-11-19T01:30:02,728 DEBUG [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 64 2023-11-19T01:30:02,728 INFO [rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357402664 on jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:02,735 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:30:02,735 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,40967,1700357333274' in zk 2023-11-19T01:30:02,739 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:30:02,740 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:02,740 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:30:02,740 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:30:02,740 DEBUG [member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:30:02,740 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:30:02,741 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:30:02,741 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:30:02,742 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:02,742 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:02,742 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:30:02,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:02,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:02,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,40967,1700357333274': 2023-11-19T01:30:02,744 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,40967,1700357333274' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:30:02,744 INFO [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:30:02,744 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:30:02,744 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:30:02,744 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:30:02,744 INFO [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:30:02,746 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:30:02,746 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:30:02,746 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:30:02,746 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:30:02,746 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:30:02,747 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:30:02,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:30:02,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:30:02,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:02,747 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:30:02,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:30:02,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:02,748 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:02,749 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:30:02,749 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:30:02,749 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:02,756 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:30:02,756 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:30:02,756 DEBUG [(jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:30:02,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:30:02,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:30:02,757 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:30:02,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:30:02,828 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357389655 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357389655 2023-11-19T01:30:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:30:02,860 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase6.apache.org:40967=1700357337536} newestTimestamps: {jenkins-hbase6.apache.org:40967=1700357389655} 2023-11-19T01:30:02,861 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357336010.meta 2023-11-19T01:30:02,861 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357356973.meta 2023-11-19T01:30:02,861 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357376277.meta 2023-11-19T01:30:02,862 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357389639.meta 2023-11-19T01:30:02,862 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta.1700357402683.meta 2023-11-19T01:30:02,862 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 2023-11-19T01:30:02,862 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357389622 2023-11-19T01:30:02,862 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357402664 2023-11-19T01:30:02,862 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 2023-11-19T01:30:02,862 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357402706 2023-11-19T01:30:03,273 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(147): Attempting to copy table info for:test-1700357341974 target: hdfs://localhost:39587/backupUT/backup_1700357401487/default/test-1700357341974 descriptor: 'test-1700357341974', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:30:03,273 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-19T01:30:03,275 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(152): Starting to write region info for table test-1700357341974 2023-11-19T01:30:03,576 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:30:03,681 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(159): Finished writing region info for table test-1700357341974 2023-11-19T01:30:04,110 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(147): Attempting to copy table info for:ns2:test-17003573419741 target: hdfs://localhost:39587/backupUT/backup_1700357401487/ns2/test-17003573419741 descriptor: 'ns2:test-17003573419741', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:30:04,110 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(149): Finished copying tableinfo. 2023-11-19T01:30:04,112 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(152): Starting to write region info for table ns2:test-17003573419741 2023-11-19T01:30:04,120 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.BackupUtils(159): Finished writing region info for table ns2:test-17003573419741 2023-11-19T01:30:04,127 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:39587/backupUT/.tmp/backup_1700357401487 from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952;hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357389622;hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991;hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258;hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294;hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357389655 2023-11-19T01:30:04,140 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c9a8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:04,141 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:30:04,143 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52804, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:30:04,145 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-19T01:30:04,145 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-17003573419741 2023-11-19T01:30:04,152 DEBUG [hconnection-0xf86dcaa-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:04,153 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39480, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:04,155 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-17003573419741: ns2:test-17003573419741; 2023-11-19T01:30:04,155 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1700357341974 2023-11-19T01:30:04,158 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1700357341974: test-1700357341974; 2023-11-19T01:30:04,158 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-11-19T01:30:04,158 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/hbase-staging/partitions_cf5fc0f4-9809-4707-bca6-8a42ee6b994f 2023-11-19T01:30:04,573 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:04,573 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:04,574 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:04,574 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,620 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop-738283206978574932.jar 2023-11-19T01:30:05,620 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,621 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,621 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,621 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,622 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,622 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,622 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,623 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,623 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,624 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:30:05,625 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:30:05,625 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:30:05,625 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:30:05,626 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:30:05,626 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:30:05,627 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:30:05,627 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:30:05,628 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:30:05,628 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:30:05,628 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:30:05,629 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:30:05,629 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,630 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,630 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,630 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,631 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,631 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:30:05,632 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:30:05,632 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-17003573419741,default:test-1700357341974 2023-11-19T01:30:05,632 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:05,633 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:30:06,687 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop-6820844808398935825.jar 2023-11-19T01:30:06,707 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 for WAL files 2023-11-19T01:30:06,708 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 2023-11-19T01:30:06,708 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357389622 for WAL files 2023-11-19T01:30:06,709 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357389622 2023-11-19T01:30:06,709 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 for WAL files 2023-11-19T01:30:06,709 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 2023-11-19T01:30:06,710 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 for WAL files 2023-11-19T01:30:06,710 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 2023-11-19T01:30:06,710 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 for WAL files 2023-11-19T01:30:06,711 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 2023-11-19T01:30:06,711 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357389655 for WAL files 2023-11-19T01:30:06,712 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357389655 2023-11-19T01:30:06,786 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.local.dir/1700357406751/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:30:06,786 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/mapreduce.cluster.local.dir/1700357406751/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:30:06,837 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 for hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357356952 (-9223372036854775808:9223372036854775807) length:18860 2023-11-19T01:30:06,842 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-19T01:30:06,842 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-19T01:30:06,891 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 for hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357376258 (-9223372036854775808:9223372036854775807) length:3657 2023-11-19T01:30:06,894 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-19T01:30:06,894 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-19T01:30:06,915 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357389622 for hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0.1700357389622 (-9223372036854775808:9223372036854775807) length:3644 2023-11-19T01:30:06,918 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-19T01:30:06,919 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-19T01:30:06,940 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 for hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/WALs/jenkins-hbase6.apache.org,40967,1700357333274/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357356991 (-9223372036854775808:9223372036854775807) length:1198 2023-11-19T01:30:06,943 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-19T01:30:06,943 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-19T01:30:06,964 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 for hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357376294 (-9223372036854775808:9223372036854775807) length:93 2023-11-19T01:30:06,967 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-19T01:30:06,967 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-19T01:30:06,988 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357389655 for hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs/jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1.1700357389655 (-9223372036854775808:9223372036854775807) length:93 2023-11-19T01:30:06,991 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-11-19T01:30:06,991 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-11-19T01:30:07,011 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b95bd0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:07,014 DEBUG [hconnection-0x3f848887-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:07,016 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39490, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:07,017 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:07,018 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-11-19T01:30:07,018 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:30:07,025 DEBUG [pool-272-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/backupUT/.tmp/backup_1700357401487/_temporary/0/_temporary/attempt_local696073222_0007_r_000000_0/ns2/test-17003573419741/f/0f668f76372b4721a0e2b32961b48cd5 is 38, key is row-t20/f:q1/1700357368147/Put/seqid=0 2023-11-19T01:30:07,027 WARN [Thread-1240 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:40967] Specified favored nodes: [jenkins-hbase6.apache.org:40967] 2023-11-19T01:30:07,454 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eafed68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:07,455 DEBUG [hconnection-0x2a27cdd5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:07,457 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39506, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:07,458 DEBUG [pool-272-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:07,461 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1700357341974-1-0] 2023-11-19T01:30:07,461 DEBUG [pool-272-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:30:07,469 DEBUG [pool-272-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/backupUT/.tmp/backup_1700357401487/_temporary/0/_temporary/attempt_local696073222_0007_r_000001_0/default/test-1700357341974/f/8d16f6cd3c084c328555f189f8e93c61 is 59, key is row-test-1700357341974-1-10/f:q1/1700357368029/Put/seqid=0 2023-11-19T01:30:07,470 WARN [Thread-1250 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:40967] Specified favored nodes: [jenkins-hbase6.apache.org:40967] 2023-11-19T01:30:08,824 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:39587/backupUT 2023-11-19T01:30:08,824 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1700357401487 2023-11-19T01:30:08,825 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-11-19T01:30:08,832 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:39587/backupUT/.tmp/backup_1700357401487, hdfs://localhost:39587/backupUT] 2023-11-19T01:30:09,495 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-11-19T01:30:09,496 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1700357401487 2023-11-19T01:30:09,497 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16003 bytes copied." 2023-11-19T01:30:09,497 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local525587713_0008 completed: true true 2023-11-19T01:30:09,500 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=5110548 FILE: Number of bytes written=9926314 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=399174 HDFS: Number of bytes written=2475126 HDFS: Number of read operations=1321 HDFS: Number of large read operations=0 HDFS: Number of write operations=412 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1708654592 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16003 Bytes Expected=16003 Files Copied=3 DIR_COPY=7 2023-11-19T01:30:09,500 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:39587/backupUT/.tmp/backup_1700357401487 to hdfs://localhost:39587/backupUT finished. 2023-11-19T01:30:09,511 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-11-19T01:30:09,511 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_4 in testing java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:160) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:30:09,511 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(240): BackupId=backup_1700357401487,startts=1700357402642,failedts=1700357409511,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_4 in testing 2023-11-19T01:30:09,511 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-11-19T01:30:09,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable backup:system 2023-11-19T01:30:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-11-19T01:30:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=76 2023-11-19T01:30:09,520 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357409520"}]},"ts":"1700357409520"} 2023-11-19T01:30:09,522 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-11-19T01:30:09,524 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-11-19T01:30:09,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, UNASSIGN}] 2023-11-19T01:30:09,526 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, UNASSIGN 2023-11-19T01:30:09,526 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=77 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:09,528 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; CloseRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:30:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=76 2023-11-19T01:30:09,679 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:09,680 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] handler.UnassignRegionHandler(119): Close 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:09,680 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1668): Closing 25b6494b58e50dbfdbcd9050bce4992b, disabling compactions & flushes 2023-11-19T01:30:09,680 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1690): Closing region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:09,680 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:09,680 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. after waiting 0 ms 2023-11-19T01:30:09,680 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:09,680 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(2819): Flushing 25b6494b58e50dbfdbcd9050bce4992b 2/2 column families, dataSize=959 B heapSize=1.87 KB 2023-11-19T01:30:09,693 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/meta/8405a319f170451cb05583ee0d61cfdb is 171, key is trslm:hdfs://localhost:39587/backupUT\x00ns2:test-17003573419741/meta:log-roll-map/1700357409504/Put/seqid=0 2023-11-19T01:30:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=76 2023-11-19T01:30:10,098 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/meta/8405a319f170451cb05583ee0d61cfdb 2023-11-19T01:30:10,106 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/222c5eb7750b488ca022402657bf4ec7 is 218, key is session:backup_1700357401487/session:context/1700357409495/Put/seqid=0 2023-11-19T01:30:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=76 2023-11-19T01:30:10,511 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/222c5eb7750b488ca022402657bf4ec7 2023-11-19T01:30:10,518 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/meta/8405a319f170451cb05583ee0d61cfdb as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/8405a319f170451cb05583ee0d61cfdb 2023-11-19T01:30:10,523 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/8405a319f170451cb05583ee0d61cfdb, entries=4, sequenceid=73, filesize=5.7 K 2023-11-19T01:30:10,524 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/222c5eb7750b488ca022402657bf4ec7 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/222c5eb7750b488ca022402657bf4ec7 2023-11-19T01:30:10,532 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/222c5eb7750b488ca022402657bf4ec7, entries=1, sequenceid=73, filesize=5.2 K 2023-11-19T01:30:10,533 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(3022): Finished flush of dataSize ~959 B/959, heapSize ~1.84 KB/1880, currentSize=0 B/0 for 25b6494b58e50dbfdbcd9050bce4992b in 853ms, sequenceid=73, compaction requested=true 2023-11-19T01:30:10,553 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/recovered.edits/76.seqid, newMaxSeqId=76, maxSeqId=59 2023-11-19T01:30:10,554 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:30:10,555 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1909): Closed backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:10,555 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1622): Region close journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:30:10,556 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] handler.UnassignRegionHandler(163): Closed 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:10,557 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=77 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=CLOSED 2023-11-19T01:30:10,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=78, resume processing ppid=77 2023-11-19T01:30:10,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=78, ppid=77, state=SUCCESS; CloseRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274 in 1.0310 sec 2023-11-19T01:30:10,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=77, resume processing ppid=76 2023-11-19T01:30:10,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=77, ppid=76, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, UNASSIGN in 1.0360 sec 2023-11-19T01:30:10,562 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357410562"}]},"ts":"1700357410562"} 2023-11-19T01:30:10,563 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-11-19T01:30:10,566 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-11-19T01:30:10,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=76, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0500 sec 2023-11-19T01:30:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=76 2023-11-19T01:30:10,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:30:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357410633 (current time:1700357410633). 2023-11-19T01:30:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:30:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1700357410632 VERSION not specified, setting to 2 2023-11-19T01:30:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f494cec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:30:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:30:10,643 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:40504, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:30:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:30:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=79, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:30:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-11-19T01:30:10,648 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:30:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-19T01:30:10,649 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:30:10,651 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot, current working dir path: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357410632 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy42.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy43.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy46.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:30:10,655 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:30:10,666 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-11-19T01:30:10,667 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-11-19T01:30:10,667 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-11-19T01:30:10,668 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:10,669 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/8405a319f170451cb05583ee0d61cfdb] hfiles 2023-11-19T01:30:10,669 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 2023-11-19T01:30:10,669 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/8405a319f170451cb05583ee0d61cfdb 2023-11-19T01:30:10,669 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:10,671 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/222c5eb7750b488ca022402657bf4ec7, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f5e1c45129834a5a845838abfd3e2dad] hfiles 2023-11-19T01:30:10,671 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/222c5eb7750b488ca022402657bf4ec7 2023-11-19T01:30:10,672 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609 2023-11-19T01:30:10,672 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c 2023-11-19T01:30:10,672 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f5e1c45129834a5a845838abfd3e2dad 2023-11-19T01:30:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-19T01:30:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-19T01:30:11,084 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-11-19T01:30:11,085 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:30:11,085 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:30:11,085 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1700357410632 2023-11-19T01:30:11,086 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357410632 2023-11-19T01:30:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-19T01:30:11,341 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-19T01:30:11,498 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:30:11,507 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:30:11,508 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1700357410632 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1700357410632 2023-11-19T01:30:11,509 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:30:11,509 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-11-19T01:30:11,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1700357410632 table=backup:system type=FLUSH ttl=0 } in 865 msec 2023-11-19T01:30:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-19T01:30:11,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=80, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357401519 type: FLUSH version: 2 ttl: 0 ) 2023-11-19T01:30:11,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-11-19T01:30:11,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=80 2023-11-19T01:30:11,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=80 2023-11-19T01:30:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=80 2023-11-19T01:30:12,183 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-11-19T01:30:12,183 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/.tabledesc/.tableinfo.0000000005.635 2023-11-19T01:30:12,184 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:30:12,186 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357401519 type: FLUSH version: 2 ttl: 0 2023-11-19T01:30:12,186 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system 2023-11-19T01:30:12,189 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-11-19T01:30:12,189 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:12,195 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/8405a319f170451cb05583ee0d61cfdb to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/archive/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/8405a319f170451cb05583ee0d61cfdb 2023-11-19T01:30:12,195 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:12,198 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/222c5eb7750b488ca022402657bf4ec7 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/archive/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/222c5eb7750b488ca022402657bf4ec7 2023-11-19T01:30:12,198 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:12,198 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357401519 type: FLUSH version: 2 ttl: 0 2023-11-19T01:30:12,199 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357412199"}]},"ts":"9223372036854775807"} 2023-11-19T01:30:12,201 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:30:12,201 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:30:12,201 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1700357412200"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357412200"}]},"ts":"1700357412200"} 2023-11-19T01:30:12,202 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:30:12,203 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-11-19T01:30:12,203 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:30:12,203 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-11-19T01:30:12,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=80, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1700357401519 type: FLUSH version: 2 ttl: 0 ) in 440 msec 2023-11-19T01:30:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=80 2023-11-19T01:30:12,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "hbase-failsafe-snapshot_backup_system-1700357410632" 2023-11-19T01:30:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1700357410632 2023-11-19T01:30:12,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.HMaster$12(2662): Client=jenkins//172.31.2.20 enable backup:system 2023-11-19T01:30:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] procedure2.ProcedureExecutor(1032): Stored pid=81, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-11-19T01:30:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-19T01:30:12,381 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-11-19T01:30:12,381 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357412381"}]},"ts":"1700357412381"} 2023-11-19T01:30:12,382 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-19T01:30:12,385 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-11-19T01:30:12,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN}] 2023-11-19T01:30:12,387 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN 2023-11-19T01:30:12,388 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-11-19T01:30:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-19T01:30:12,540 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:12,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274}] 2023-11-19T01:30:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-19T01:30:12,694 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:12,698 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(130): Open backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:12,698 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7254): Opening region: {ENCODED => 25b6494b58e50dbfdbcd9050bce4992b, NAME => 'backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:30:12,699 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:30:12,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:12,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(888): Instantiated backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:30:12,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7296): checking encryption for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:12,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7299): checking classloading for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:12,701 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:12,702 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25b6494b58e50dbfdbcd9050bce4992b columnFamilyName meta 2023-11-19T01:30:12,702 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:12,713 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/meta/4eca121b9c7643b7b78ca1d3b2b74ce1 2023-11-19T01:30:12,713 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(324): Store=25b6494b58e50dbfdbcd9050bce4992b/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:12,713 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:12,715 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25b6494b58e50dbfdbcd9050bce4992b columnFamilyName session 2023-11-19T01:30:12,715 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:30:12,724 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609 2023-11-19T01:30:12,729 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c 2023-11-19T01:30:12,738 DEBUG [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f5e1c45129834a5a845838abfd3e2dad 2023-11-19T01:30:12,738 INFO [StoreOpener-25b6494b58e50dbfdbcd9050bce4992b-1 {}] regionserver.HStore(324): Store=25b6494b58e50dbfdbcd9050bce4992b/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:30:12,740 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:12,741 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:12,743 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:30:12,780 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1076): writing seq id for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:12,781 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): Opened 25b6494b58e50dbfdbcd9050bce4992b; next sequenceid=77; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11845576320, jitterRate=0.10320526361465454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:30:12,782 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(992): Region open journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:30:12,785 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b., pid=83, masterSystemTime=1700357412694 2023-11-19T01:30:12,786 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.CompactSplit(403): Add compact mark for store 25b6494b58e50dbfdbcd9050bce4992b:session, priority=-2147483648, current under compaction store size is 1 2023-11-19T01:30:12,787 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-11-19T01:30:12,787 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-11-19T01:30:12,789 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=25b6494b58e50dbfdbcd9050bce4992b, regionState=OPEN, openSeqNum=77, regionLocation=jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:12,790 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:12,790 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(158): Opened backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:12,790 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 16834 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-11-19T01:30:12,790 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.HStore(1534): 25b6494b58e50dbfdbcd9050bce4992b/session is initiating minor compaction (all files) 2023-11-19T01:30:12,790 INFO [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 25b6494b58e50dbfdbcd9050bce4992b/session in backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:12,790 INFO [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f5e1c45129834a5a845838abfd3e2dad] into tmpdir=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp, totalSize=16.4 K 2023-11-19T01:30:12,791 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] compactions.Compactor(223): Compacting 570e5efe080c4c40a868ff08386fd609, keycount=4, bloomtype=ROW, size=6.0 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1700357367978 2023-11-19T01:30:12,792 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] compactions.Compactor(223): Compacting f137c14c96c8462d80e28a3ba7fb085c, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1700357387386 2023-11-19T01:30:12,792 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] compactions.Compactor(223): Compacting f5e1c45129834a5a845838abfd3e2dad, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1700357401417 2023-11-19T01:30:12,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=83, resume processing ppid=82 2023-11-19T01:30:12,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure 25b6494b58e50dbfdbcd9050bce4992b, server=jenkins-hbase6.apache.org,40967,1700357333274 in 250 msec 2023-11-19T01:30:12,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=82, resume processing ppid=81 2023-11-19T01:30:12,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=25b6494b58e50dbfdbcd9050bce4992b, ASSIGN in 408 msec 2023-11-19T01:30:12,796 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357412796"}]},"ts":"1700357412796"} 2023-11-19T01:30:12,798 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-19T01:30:12,801 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-11-19T01:30:12,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=81, state=SUCCESS; EnableTableProcedure table=backup:system in 425 msec 2023-11-19T01:30:12,830 INFO [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 25b6494b58e50dbfdbcd9050bce4992b#session#compaction#17 average throughput is 1.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-11-19T01:30:12,831 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/788083eeca3149f1869bb2aaba6c4248 is 459, key is session:backup_1700357350318/session:context/1700357367978/Put/seqid=0 2023-11-19T01:30:12,856 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/788083eeca3149f1869bb2aaba6c4248 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/788083eeca3149f1869bb2aaba6c4248 2023-11-19T01:30:12,867 INFO [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 25b6494b58e50dbfdbcd9050bce4992b/session of 25b6494b58e50dbfdbcd9050bce4992b into 788083eeca3149f1869bb2aaba6c4248(size=6.5 K), total size for store is 6.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-11-19T01:30:12,867 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:30:12,867 INFO [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b., storeName=25b6494b58e50dbfdbcd9050bce4992b/session, priority=13, startTime=1700357412785; duration=0sec 2023-11-19T01:30:12,867 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-11-19T01:30:12,867 DEBUG [RS:0;jenkins-hbase6:40967-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 25b6494b58e50dbfdbcd9050bce4992b:session 2023-11-19T01:30:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-19T01:30:12,985 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-11-19T01:30:12,985 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:30:12,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:30:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36603 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:30:12,994 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:30:12,994 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-11-19T01:30:12,995 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:39587/backupUT/backup_1700357401487/default/test-1700357341974 done. 2023-11-19T01:30:12,996 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(208): hdfs://localhost:39587/backupUT/backup_1700357401487/default is empty, remove it. 2023-11-19T01:30:12,996 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:39587/backupUT/backup_1700357401487/ns2/test-17003573419741 done. 2023-11-19T01:30:12,997 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(208): hdfs://localhost:39587/backupUT/backup_1700357401487/ns2 is empty, remove it. 2023-11-19T01:30:12,999 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:30:13,006 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] impl.TableBackupClient(246): Backup backup_1700357401487 failed. Backup session finished. Status: FAILURE 2023-11-19T01:30:13,006 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:13,006 ERROR [Listener at jenkins-hbase6.apache.org/43675 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:160) ~[test-classes/:?] ... 41 more 2023-11-19T01:30:13,012 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:13,032 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackupWithFailures#testIncBackupRestore[0] Thread=602 (was 587) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS:0;jenkins-hbase6:40967-shortCompactions-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x625798ee-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:59682@0x7740a47b-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/cluster_e2bb288f-7ce7-cac6-514c-081c116c782a/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x5b0d6c55-shared-pool-32 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1557001192_16 at /127.0.0.1:48636 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase6.apache.org,36603,1700357330230)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_97094098_16 at /127.0.0.1:41768 [Receiving block BP-2068368082-172.31.2.20-1700357327020:blk_1073741940_1116] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_97094098_16 at /127.0.0.1:41742 [Receiving block BP-2068368082-172.31.2.20-1700357327020:blk_1073741938_1114] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1557001192_16 at /127.0.0.1:48640 [Waiting for operation #6] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/cluster_e2bb288f-7ce7-cac6-514c-081c116c782a/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_97094098_16 at /127.0.0.1:41752 [Receiving block BP-2068368082-172.31.2.20-1700357327020:blk_1073741939_1115] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:59682@0x7740a47b-SendThread(127.0.0.1:59682) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1394) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: PacketResponder: BP-2068368082-172.31.2.20-1700357327020:blk_1073741940_1116, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x625798ee-shared-pool-25 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2068368082-172.31.2.20-1700357327020:blk_1073741938_1114, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1557001192_16 at /127.0.0.1:48654 [Waiting for operation #6] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2068368082-172.31.2.20-1700357327020:blk_1073741939_1115, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1557001192_16 at /127.0.0.1:41826 [Waiting for operation #32] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase6.apache.org,40967,1700357333274)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase6.apache.org,40967,1700357333274' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1557001192_16 at /127.0.0.1:48616 [Waiting for operation #6] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1557001192_16 at /127.0.0.1:48584 [Waiting for operation #10] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:59682@0x7740a47b java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1517) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1541) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$532/491590983.run(Unknown Source) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=716 (was 696) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=156 (was 124) - SystemLoadAverage LEAK? -, ProcessCount=166 (was 166), AvailableMemoryMB=3633 (was 3586) - AvailableMemoryMB LEAK? - 2023-11-19T01:30:13,033 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-11-19T01:30:13,043 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-11-19T01:30:13,044 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:13,044 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-11-19T01:30:13,045 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] util.JVMClusterUtil(257): Found active master hash=1514954901, stopped=false 2023-11-19T01:30:13,045 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] master.ServerManager(896): Cluster shutdown requested of master=jenkins-hbase6.apache.org,36603,1700357330230 2023-11-19T01:30:13,048 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] procedure2.ProcedureExecutor(630): Stopping 2023-11-19T01:30:13,049 DEBUG [Listener at jenkins-hbase6.apache.org/43675 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:13,049 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase6.apache.org,40967,1700357333274' ***** 2023-11-19T01:30:13,049 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-11-19T01:30:13,049 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-19T01:30:13,066 INFO [RS:0;jenkins-hbase6:40967 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@110ad48e{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-19T01:30:13,071 INFO [RS:0;jenkins-hbase6:40967 {}] server.AbstractConnector(383): Stopped ServerConnector@4dfe1c86{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:30:13,071 INFO [RS:0;jenkins-hbase6:40967 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-19T01:30:13,072 INFO [RS:0;jenkins-hbase6:40967 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1355a3b7{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-19T01:30:13,072 INFO [RS:0;jenkins-hbase6:40967 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3040caf3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop.log.dir/,STOPPED} 2023-11-19T01:30:13,078 INFO [regionserver/jenkins-hbase6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-11-19T01:30:13,078 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-11-19T01:30:13,082 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HeapMemoryManager(220): Stopping 2023-11-19T01:30:13,082 INFO [RS:0;jenkins-hbase6:40967 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-11-19T01:30:13,082 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-11-19T01:30:13,082 INFO [RS:0;jenkins-hbase6:40967 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-11-19T01:30:13,082 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(3528): Received CLOSE for 2eb9f9fa54beda37677c07a7a5b64fe7 2023-11-19T01:30:13,083 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(3528): Received CLOSE for 4626e266aa32d68d09dea77ed5ea495c 2023-11-19T01:30:13,083 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(3528): Received CLOSE for 36f383735ae1fecc54a98975fe8d38af 2023-11-19T01:30:13,083 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(3528): Received CLOSE for 4b19ffa2d6c4736cdf4abc8f26e2bae3 2023-11-19T01:30:13,083 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(3528): Received CLOSE for 25b6494b58e50dbfdbcd9050bce4992b 2023-11-19T01:30:13,083 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(3528): Received CLOSE for 587767aa1f479d2c3ca4e5abbe608c38 2023-11-19T01:30:13,083 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(3528): Received CLOSE for 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:30:13,083 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase6.apache.org,40967,1700357333274 2023-11-19T01:30:13,083 DEBUG [RS:0;jenkins-hbase6:40967 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:13,083 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-11-19T01:30:13,083 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-11-19T01:30:13,083 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-11-19T01:30:13,083 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-11-19T01:30:13,084 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 2eb9f9fa54beda37677c07a7a5b64fe7, disabling compactions & flushes 2023-11-19T01:30:13,084 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. 2023-11-19T01:30:13,084 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. 2023-11-19T01:30:13,084 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. after waiting 0 ms 2023-11-19T01:30:13,084 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. 2023-11-19T01:30:13,084 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-11-19T01:30:13,084 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1583): Online Regions={2eb9f9fa54beda37677c07a7a5b64fe7=backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7., 4626e266aa32d68d09dea77ed5ea495c=ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c., 36f383735ae1fecc54a98975fe8d38af=hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af., 4b19ffa2d6c4736cdf4abc8f26e2bae3=ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3., 1588230740=hbase:meta,,1.1588230740, 25b6494b58e50dbfdbcd9050bce4992b=backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b., 587767aa1f479d2c3ca4e5abbe608c38=ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38., 3c282db26b60e0d893a6e12adb946977=test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977.} 2023-11-19T01:30:13,085 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-19T01:30:13,085 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-19T01:30:13,085 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-19T01:30:13,086 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-19T01:30:13,086 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-19T01:30:13,086 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=18.81 KB heapSize=32.60 KB 2023-11-19T01:30:13,086 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 25b6494b58e50dbfdbcd9050bce4992b, 2eb9f9fa54beda37677c07a7a5b64fe7, 36f383735ae1fecc54a98975fe8d38af, 3c282db26b60e0d893a6e12adb946977, 4626e266aa32d68d09dea77ed5ea495c, 4b19ffa2d6c4736cdf4abc8f26e2bae3, 587767aa1f479d2c3ca4e5abbe608c38 2023-11-19T01:30:13,095 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system_bulk/2eb9f9fa54beda37677c07a7a5b64fe7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-19T01:30:13,096 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:30:13,097 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. 2023-11-19T01:30:13,097 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 2eb9f9fa54beda37677c07a7a5b64fe7: 2023-11-19T01:30:13,097 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1700357352595.2eb9f9fa54beda37677c07a7a5b64fe7. 2023-11-19T01:30:13,097 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 4626e266aa32d68d09dea77ed5ea495c, disabling compactions & flushes 2023-11-19T01:30:13,097 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:30:13,097 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:30:13,097 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. after waiting 0 ms 2023-11-19T01:30:13,097 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:30:13,097 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 4626e266aa32d68d09dea77ed5ea495c 1/1 column families, dataSize=170 B heapSize=816 B 2023-11-19T01:30:13,101 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/.tmp/f/9fea708c5ca243aba163f3e2eac530c7 is 38, key is row-t20/f:q1/1700357368147/Put/seqid=0 2023-11-19T01:30:13,116 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/.tmp/f/9fea708c5ca243aba163f3e2eac530c7 2023-11-19T01:30:13,125 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/.tmp/f/9fea708c5ca243aba163f3e2eac530c7 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/f/9fea708c5ca243aba163f3e2eac530c7 2023-11-19T01:30:13,126 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740/.tmp/info/b7558ae2a4b949edbb7b2b7dfd5f04cc is 159, key is ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c./info:regioninfo/1700357345096/Put/seqid=0 2023-11-19T01:30:13,132 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/f/9fea708c5ca243aba163f3e2eac530c7, entries=5, sequenceid=111, filesize=5.0 K 2023-11-19T01:30:13,133 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for 4626e266aa32d68d09dea77ed5ea495c in 35ms, sequenceid=111, compaction requested=false 2023-11-19T01:30:13,141 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.41 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740/.tmp/info/b7558ae2a4b949edbb7b2b7dfd5f04cc 2023-11-19T01:30:13,148 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns2/test-17003573419741/4626e266aa32d68d09dea77ed5ea495c/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2023-11-19T01:30:13,150 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:30:13,151 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:30:13,151 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 4626e266aa32d68d09dea77ed5ea495c: 2023-11-19T01:30:13,151 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17003573419741,,1700357344706.4626e266aa32d68d09dea77ed5ea495c. 2023-11-19T01:30:13,153 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 36f383735ae1fecc54a98975fe8d38af, disabling compactions & flushes 2023-11-19T01:30:13,153 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. 2023-11-19T01:30:13,153 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. 2023-11-19T01:30:13,153 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. after waiting 0 ms 2023-11-19T01:30:13,153 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. 2023-11-19T01:30:13,153 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 36f383735ae1fecc54a98975fe8d38af 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-11-19T01:30:13,170 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740/.tmp/table/5c6e0035f2bd4844a7ca6005689d687d is 59, key is ns2:test-17003573419741/table:state/1700357345106/Put/seqid=0 2023-11-19T01:30:13,172 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/namespace/36f383735ae1fecc54a98975fe8d38af/.tmp/info/ca22176581b745beb76367ffe1f70355 is 45, key is default/info:d/1700357337671/Put/seqid=0 2023-11-19T01:30:13,185 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.40 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740/.tmp/table/5c6e0035f2bd4844a7ca6005689d687d 2023-11-19T01:30:13,191 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740/.tmp/info/b7558ae2a4b949edbb7b2b7dfd5f04cc as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740/info/b7558ae2a4b949edbb7b2b7dfd5f04cc 2023-11-19T01:30:13,197 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740/info/b7558ae2a4b949edbb7b2b7dfd5f04cc, entries=79, sequenceid=83, filesize=15.1 K 2023-11-19T01:30:13,198 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740/.tmp/table/5c6e0035f2bd4844a7ca6005689d687d as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740/table/5c6e0035f2bd4844a7ca6005689d687d 2023-11-19T01:30:13,204 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740/table/5c6e0035f2bd4844a7ca6005689d687d, entries=15, sequenceid=83, filesize=5.8 K 2023-11-19T01:30:13,205 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~18.81 KB/19263, heapSize ~32.32 KB/33096, currentSize=0 B/0 for 1588230740 in 119ms, sequenceid=83, compaction requested=false 2023-11-19T01:30:13,215 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/meta/1588230740/recovered.edits/86.seqid, newMaxSeqId=86, maxSeqId=1 2023-11-19T01:30:13,216 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:30:13,216 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-19T01:30:13,216 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-19T01:30:13,216 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-19T01:30:13,217 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-11-19T01:30:13,286 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1609): Waiting on 25b6494b58e50dbfdbcd9050bce4992b, 36f383735ae1fecc54a98975fe8d38af, 3c282db26b60e0d893a6e12adb946977, 4b19ffa2d6c4736cdf4abc8f26e2bae3, 587767aa1f479d2c3ca4e5abbe608c38 2023-11-19T01:30:13,486 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1609): Waiting on 25b6494b58e50dbfdbcd9050bce4992b, 36f383735ae1fecc54a98975fe8d38af, 3c282db26b60e0d893a6e12adb946977, 4b19ffa2d6c4736cdf4abc8f26e2bae3, 587767aa1f479d2c3ca4e5abbe608c38 2023-11-19T01:30:13,587 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/namespace/36f383735ae1fecc54a98975fe8d38af/.tmp/info/ca22176581b745beb76367ffe1f70355 2023-11-19T01:30:13,592 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/namespace/36f383735ae1fecc54a98975fe8d38af/.tmp/info/ca22176581b745beb76367ffe1f70355 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/namespace/36f383735ae1fecc54a98975fe8d38af/info/ca22176581b745beb76367ffe1f70355 2023-11-19T01:30:13,597 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/namespace/36f383735ae1fecc54a98975fe8d38af/info/ca22176581b745beb76367ffe1f70355, entries=7, sequenceid=11, filesize=5.1 K 2023-11-19T01:30:13,597 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 36f383735ae1fecc54a98975fe8d38af in 444ms, sequenceid=11, compaction requested=false 2023-11-19T01:30:13,603 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/hbase/namespace/36f383735ae1fecc54a98975fe8d38af/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-11-19T01:30:13,604 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:30:13,604 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. 2023-11-19T01:30:13,604 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 36f383735ae1fecc54a98975fe8d38af: 2023-11-19T01:30:13,604 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1700357336297.36f383735ae1fecc54a98975fe8d38af. 2023-11-19T01:30:13,605 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 4b19ffa2d6c4736cdf4abc8f26e2bae3, disabling compactions & flushes 2023-11-19T01:30:13,605 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. 2023-11-19T01:30:13,605 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. 2023-11-19T01:30:13,605 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. after waiting 0 ms 2023-11-19T01:30:13,605 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. 2023-11-19T01:30:13,608 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns4/test-17003573419743/4b19ffa2d6c4736cdf4abc8f26e2bae3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-19T01:30:13,609 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:30:13,609 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. 2023-11-19T01:30:13,609 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 4b19ffa2d6c4736cdf4abc8f26e2bae3: 2023-11-19T01:30:13,609 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17003573419743,,1700357347538.4b19ffa2d6c4736cdf4abc8f26e2bae3. 2023-11-19T01:30:13,609 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 25b6494b58e50dbfdbcd9050bce4992b, disabling compactions & flushes 2023-11-19T01:30:13,609 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:13,609 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:13,610 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. after waiting 0 ms 2023-11-19T01:30:13,610 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:13,610 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 25b6494b58e50dbfdbcd9050bce4992b 2/2 column families, dataSize=296 B heapSize=960 B 2023-11-19T01:30:13,613 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/b215887d37134ed1b6b9296b2ca1e626 is 256, key is session:backup_1700357401487/session:context/1700357412998/Put/seqid=0 2023-11-19T01:30:13,687 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1609): Waiting on 25b6494b58e50dbfdbcd9050bce4992b, 3c282db26b60e0d893a6e12adb946977, 587767aa1f479d2c3ca4e5abbe608c38 2023-11-19T01:30:13,887 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1609): Waiting on 25b6494b58e50dbfdbcd9050bce4992b, 3c282db26b60e0d893a6e12adb946977, 587767aa1f479d2c3ca4e5abbe608c38 2023-11-19T01:30:13,894 INFO [regionserver/jenkins-hbase6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-11-19T01:30:13,894 INFO [regionserver/jenkins-hbase6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-11-19T01:30:14,018 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=296 B at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/b215887d37134ed1b6b9296b2ca1e626 2023-11-19T01:30:14,024 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/.tmp/session/b215887d37134ed1b6b9296b2ca1e626 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/b215887d37134ed1b6b9296b2ca1e626 2023-11-19T01:30:14,028 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/b215887d37134ed1b6b9296b2ca1e626, entries=2, sequenceid=82, filesize=5.2 K 2023-11-19T01:30:14,029 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~296 B/296, heapSize ~688 B/688, currentSize=0 B/0 for 25b6494b58e50dbfdbcd9050bce4992b in 419ms, sequenceid=82, compaction requested=false 2023-11-19T01:30:14,036 DEBUG [StoreCloser-backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f5e1c45129834a5a845838abfd3e2dad] to archive 2023-11-19T01:30:14,037 DEBUG [StoreCloser-backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-11-19T01:30:14,038 DEBUG [StoreCloser-backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609 to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/archive/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/570e5efe080c4c40a868ff08386fd609 2023-11-19T01:30:14,040 DEBUG [StoreCloser-backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/archive/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f137c14c96c8462d80e28a3ba7fb085c 2023-11-19T01:30:14,041 DEBUG [StoreCloser-backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f5e1c45129834a5a845838abfd3e2dad to hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/archive/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/session/f5e1c45129834a5a845838abfd3e2dad 2023-11-19T01:30:14,048 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/backup/system/25b6494b58e50dbfdbcd9050bce4992b/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=76 2023-11-19T01:30:14,048 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:30:14,048 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:14,048 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 25b6494b58e50dbfdbcd9050bce4992b: 2023-11-19T01:30:14,048 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1700357350467.25b6494b58e50dbfdbcd9050bce4992b. 2023-11-19T01:30:14,049 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 587767aa1f479d2c3ca4e5abbe608c38, disabling compactions & flushes 2023-11-19T01:30:14,049 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. 2023-11-19T01:30:14,049 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. 2023-11-19T01:30:14,049 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. after waiting 0 ms 2023-11-19T01:30:14,049 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. 2023-11-19T01:30:14,052 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/ns3/test-17003573419742/587767aa1f479d2c3ca4e5abbe608c38/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-19T01:30:14,053 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:30:14,054 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. 2023-11-19T01:30:14,054 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 587767aa1f479d2c3ca4e5abbe608c38: 2023-11-19T01:30:14,054 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17003573419742,,1700357345409.587767aa1f479d2c3ca4e5abbe608c38. 2023-11-19T01:30:14,054 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 3c282db26b60e0d893a6e12adb946977, disabling compactions & flushes 2023-11-19T01:30:14,054 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:30:14,054 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:30:14,054 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. after waiting 0 ms 2023-11-19T01:30:14,054 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:30:14,054 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 3c282db26b60e0d893a6e12adb946977 2/2 column families, dataSize=5.30 KB heapSize=12.88 KB 2023-11-19T01:30:14,068 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/.tmp/f/38d703086636421ab700f8518e621654 is 59, key is row-test-1700357341974-1-10/f:q1/1700357368029/Put/seqid=0 2023-11-19T01:30:14,087 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1579): Waiting on 1 regions to close 2023-11-19T01:30:14,087 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1583): Online Regions={3c282db26b60e0d893a6e12adb946977=test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977.} 2023-11-19T01:30:14,087 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1609): Waiting on 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:30:14,287 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1609): Waiting on 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:30:14,472 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/.tmp/f/38d703086636421ab700f8518e621654 2023-11-19T01:30:14,479 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/.tmp/f/38d703086636421ab700f8518e621654 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f/38d703086636421ab700f8518e621654 2023-11-19T01:30:14,485 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/f/38d703086636421ab700f8518e621654, entries=99, sequenceid=217, filesize=10.5 K 2023-11-19T01:30:14,486 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~5.30 KB/5425, heapSize ~12.61 KB/12912, currentSize=0 B/0 for 3c282db26b60e0d893a6e12adb946977 in 432ms, sequenceid=217, compaction requested=false 2023-11-19T01:30:14,489 DEBUG [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1609): Waiting on 3c282db26b60e0d893a6e12adb946977 2023-11-19T01:30:14,497 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/data/default/test-1700357341974/3c282db26b60e0d893a6e12adb946977/recovered.edits/220.seqid, newMaxSeqId=220, maxSeqId=106 2023-11-19T01:30:14,497 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:30:14,498 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:30:14,498 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 3c282db26b60e0d893a6e12adb946977: 2023-11-19T01:30:14,498 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1700357341974,,1700357342465.3c282db26b60e0d893a6e12adb946977. 2023-11-19T01:30:14,689 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase6.apache.org,40967,1700357333274; all regions closed. 2023-11-19T01:30:14,699 DEBUG [RS:0;jenkins-hbase6:40967 {}] wal.AbstractFSWAL(1062): Moved 5 WAL file(s) to /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs 2023-11-19T01:30:14,699 INFO [RS:0;jenkins-hbase6:40967 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.meta:.meta(num 1700357402683) 2023-11-19T01:30:14,704 DEBUG [RS:0;jenkins-hbase6:40967 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs 2023-11-19T01:30:14,704 INFO [RS:0;jenkins-hbase6:40967 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-0:(num 1700357402664) 2023-11-19T01:30:14,710 DEBUG [RS:0;jenkins-hbase6:40967 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/oldWALs 2023-11-19T01:30:14,710 INFO [RS:0;jenkins-hbase6:40967 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase6.apache.org%2C40967%2C1700357333274.jenkins-hbase6.apache.org%2C40967%2C1700357333274.regiongroup-1:(num 1700357402706) 2023-11-19T01:30:14,710 DEBUG [RS:0;jenkins-hbase6:40967 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:14,710 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.LeaseManager(133): Closed leases 2023-11-19T01:30:14,710 INFO [RS:0;jenkins-hbase6:40967 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-11-19T01:30:14,710 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-19T01:30:14,712 INFO [RS:0;jenkins-hbase6:40967 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.2.20:40967 2023-11-19T01:30:14,723 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase6.apache.org,40967,1700357333274] 2023-11-19T01:30:14,723 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase6.apache.org,40967,1700357333274; numProcessing=1 2023-11-19T01:30:14,724 INFO [RegionServerTracker-0 {}] master.ServerManager(561): Cluster shutdown set; jenkins-hbase6.apache.org,40967,1700357333274 expired; onlineServers=0 2023-11-19T01:30:14,724 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase6.apache.org,36603,1700357330230' ***** 2023-11-19T01:30:14,725 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-11-19T01:30:14,725 DEBUG [M:0;jenkins-hbase6:36603 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c08fe89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase6.apache.org/172.31.2.20:0 2023-11-19T01:30:14,725 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-19T01:30:14,770 INFO [M:0;jenkins-hbase6:36603 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@52673c48{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-19T01:30:14,771 INFO [M:0;jenkins-hbase6:36603 {}] server.AbstractConnector(383): Stopped ServerConnector@48dc6477{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:30:14,771 INFO [M:0;jenkins-hbase6:36603 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-19T01:30:14,772 INFO [M:0;jenkins-hbase6:36603 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@68d66e26{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-19T01:30:14,772 INFO [M:0;jenkins-hbase6:36603 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@50df4d49{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/hadoop.log.dir/,STOPPED} 2023-11-19T01:30:14,773 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase6.apache.org,36603,1700357330230 2023-11-19T01:30:14,773 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase6.apache.org,36603,1700357330230; all regions closed. 2023-11-19T01:30:14,773 DEBUG [M:0;jenkins-hbase6:36603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:14,773 INFO [M:0;jenkins-hbase6:36603 {}] master.HMaster(1666): Stopping master jetty server 2023-11-19T01:30:14,774 INFO [M:0;jenkins-hbase6:36603 {}] server.AbstractConnector(383): Stopped ServerConnector@1ea10a7a{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:30:14,774 DEBUG [M:0;jenkins-hbase6:36603 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-11-19T01:30:14,774 INFO [M:0;jenkins-hbase6:36603 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-11-19T01:30:14,774 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-11-19T01:30:14,774 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357334613 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357334613,5,FailOnTimeoutGroup] 2023-11-19T01:30:14,774 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357334612 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357334612,5,FailOnTimeoutGroup] 2023-11-19T01:30:14,774 DEBUG [M:0;jenkins-hbase6:36603 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:30:14,774 DEBUG [M:0;jenkins-hbase6:36603 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-11-19T01:30:14,775 INFO [M:0;jenkins-hbase6:36603 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase6:0 had [] on shutdown 2023-11-19T01:30:14,775 DEBUG [M:0;jenkins-hbase6:36603 {}] master.HMaster(1689): Stopping service threads 2023-11-19T01:30:14,775 INFO [M:0;jenkins-hbase6:36603 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-11-19T01:30:14,775 ERROR [M:0;jenkins-hbase6:36603 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-11-19T01:30:14,776 INFO [M:0;jenkins-hbase6:36603 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-11-19T01:30:14,776 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-11-19T01:30:14,776 WARN [M:0;jenkins-hbase6:36603 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-11-19T01:30:14,776 INFO [M:0;jenkins-hbase6:36603 {}] assignment.AssignmentManager(383): Stopping assignment manager 2023-11-19T01:30:14,776 INFO [M:0;jenkins-hbase6:36603 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-11-19T01:30:14,777 DEBUG [M:0;jenkins-hbase6:36603 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-19T01:30:14,789 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:30:14,789 DEBUG [M:0;jenkins-hbase6:36603 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:30:14,789 DEBUG [M:0;jenkins-hbase6:36603 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-19T01:30:14,789 DEBUG [M:0;jenkins-hbase6:36603 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:30:14,789 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=326.06 KB heapSize=389.77 KB 2023-11-19T01:30:14,801 DEBUG [M:0;jenkins-hbase6:36603 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/34617a1692f044a0966b391e22cf151a is 88, key is hbase:meta,,1/info:sn/1700357335620/Put/seqid=0 2023-11-19T01:30:14,822 INFO [RS:0;jenkins-hbase6:40967 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase6.apache.org,40967,1700357333274; zookeeper connection closed. 2023-11-19T01:30:14,823 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2d64eb7 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2d64eb7 2023-11-19T01:30:14,823 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-11-19T01:30:15,206 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=758 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/34617a1692f044a0966b391e22cf151a 2023-11-19T01:30:15,225 DEBUG [M:0;jenkins-hbase6:36603 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b885c6b841dd4beb8f59450d62ba6073 is 1281, key is \x00\x00\x00\x00\x00\x00\x00\x19/proc:d/1700357350281/Put/seqid=0 2023-11-19T01:30:15,629 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=325.47 KB at sequenceid=758 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b885c6b841dd4beb8f59450d62ba6073 2023-11-19T01:30:15,635 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b885c6b841dd4beb8f59450d62ba6073 2023-11-19T01:30:15,650 DEBUG [M:0;jenkins-hbase6:36603 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4db755b29b2a46a488a7446e4c62aa49 is 82, key is jenkins-hbase6.apache.org,40967,1700357333274/rs:state/1700357334755/Put/seqid=0 2023-11-19T01:30:16,055 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=758 (bloomFilter=true), to=hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4db755b29b2a46a488a7446e4c62aa49 2023-11-19T01:30:16,061 DEBUG [M:0;jenkins-hbase6:36603 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/34617a1692f044a0966b391e22cf151a as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/34617a1692f044a0966b391e22cf151a 2023-11-19T01:30:16,066 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/34617a1692f044a0966b391e22cf151a, entries=8, sequenceid=758, filesize=5.5 K 2023-11-19T01:30:16,067 DEBUG [M:0;jenkins-hbase6:36603 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b885c6b841dd4beb8f59450d62ba6073 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b885c6b841dd4beb8f59450d62ba6073 2023-11-19T01:30:16,072 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b885c6b841dd4beb8f59450d62ba6073 2023-11-19T01:30:16,072 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b885c6b841dd4beb8f59450d62ba6073, entries=83, sequenceid=758, filesize=26.0 K 2023-11-19T01:30:16,073 DEBUG [M:0;jenkins-hbase6:36603 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4db755b29b2a46a488a7446e4c62aa49 as hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4db755b29b2a46a488a7446e4c62aa49 2023-11-19T01:30:16,077 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:39587/user/jenkins/test-data/dddd58a7-ca14-9f81-fda6-73bfcbb2c5fa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4db755b29b2a46a488a7446e4c62aa49, entries=1, sequenceid=758, filesize=5.1 K 2023-11-19T01:30:16,078 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.HRegion(3022): Finished flush of dataSize ~326.06 KB/333890, heapSize ~389.48 KB/398824, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1289ms, sequenceid=758, compaction requested=false 2023-11-19T01:30:16,083 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:30:16,083 DEBUG [M:0;jenkins-hbase6:36603 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:30:16,087 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-19T01:30:16,087 INFO [M:0;jenkins-hbase6:36603 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-11-19T01:30:16,087 INFO [M:0;jenkins-hbase6:36603 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-11-19T01:30:16,088 INFO [M:0;jenkins-hbase6:36603 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.2.20:36603 2023-11-19T01:30:16,101 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:30:16,193 INFO [M:0;jenkins-hbase6:36603 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase6.apache.org,36603,1700357330230; zookeeper connection closed. 2023-11-19T01:30:16,198 WARN [Listener at jenkins-hbase6.apache.org/43675 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-19T01:30:16,203 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-19T01:30:16,310 WARN [BP-2068368082-172.31.2.20-1700357327020 heartbeating to localhost/127.0.0.1:39587 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-19T01:30:16,310 WARN [BP-2068368082-172.31.2.20-1700357327020 heartbeating to localhost/127.0.0.1:39587 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-2068368082-172.31.2.20-1700357327020 (Datanode Uuid fe2559e6-db39-4175-a0c5-755883551972) service to localhost/127.0.0.1:39587 2023-11-19T01:30:16,312 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/cluster_e2bb288f-7ce7-cac6-514c-081c116c782a/dfs/data/data1/current/BP-2068368082-172.31.2.20-1700357327020 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:30:16,313 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/dc68edeb-754e-d480-be17-356f0f16b012/cluster_e2bb288f-7ce7-cac6-514c-081c116c782a/dfs/data/data2/current/BP-2068368082-172.31.2.20-1700357327020 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:30:16,336 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-19T01:30:16,514 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-11-19T01:30:16,515 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-11-19T01:30:16,525 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:30:16,639 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:30:16,751 ERROR [Thread[Thread-277,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-19T01:30:16,751 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:30:16,856 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-11-19T01:30:16,862 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-11-19T01:30:16,864 ERROR [Thread[Thread-287,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-19T01:30:16,867 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:30:16,970 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-19T01:30:16,970 INFO [Listener at jenkins-hbase6.apache.org/43675 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped